GPU:        NVIDIA GeForce RTX 5090
NNlib:      0.9.47
LuxLib:     1.15.9
CUDA.jl:    6.4.2
cuDNN:      6.4.2 (library 9.26.0)
selection:  dtypes=Float16,BFloat16,Float32  impls=nnlib,nnlib-ka,nnlib-mat,lux  causal=false,true  sizes=tiny,small,gpt2-ish,llama3 L=2k,llama3 L=4k,llama3 L=8k
sanity Float16    (vs nnlib)  nnlib=0.00e+00  nnlib-ka=4.88e-04  nnlib-mat=2.44e-03  lux=2.44e-03
sanity BFloat16   (vs nnlib)  nnlib=0.00e+00  nnlib-ka=7.81e-03  nnlib-mat=6.25e-02  lux=6.25e-02
sanity Float32    (vs nnlib)  nnlib=0.00e+00  nnlib-ka=0.00e+00  nnlib-mat=4.17e-07  lux=4.17e-07

########################################################################################################
### dtype=Float16  causal=false
########################################################################################################

-- FORWARD (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.013      0.025      0.753      0.755 |            1.87x           56.42x           56.53x
small           512   8 |      0.036      0.204      1.451      1.462 |            5.60x           39.87x           40.17x
gpt2-ish       1024  12 |      0.118      0.506      1.103      1.095 |            4.29x            9.36x            9.29x
llama3 L=2k    2048  32 |      0.407      2.288      1.533      1.538 |            5.62x            3.76x            3.77x
llama3 L=4k    4096  32 |      1.388      8.410      4.161      4.179 |            6.06x            3.00x            3.01x
llama3 L=8k    8192  32 |      5.126     31.814        n/s        n/s |            6.21x                -                -

-- FULL GRADIENT, fwd+bwd (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.168      0.145      2.683      3.255 |            0.86x           16.00x           19.42x
small           512   8 |      0.198      1.045      5.172      5.429 |            5.27x           26.10x           27.39x
gpt2-ish       1024  12 |      0.454      2.569      3.992      4.466 |            5.66x            8.79x            9.84x
llama3 L=2k    2048  32 |      1.561     11.557      4.627      4.564 |            7.40x            2.96x            2.92x
llama3 L=4k    4096  32 |      5.352     41.514     11.799     11.582 |            7.76x            2.20x            2.16x
llama3 L=8k    8192  32 |     19.270    156.423        n/s        n/s |            8.12x                -                -

-- FORWARD, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.500      0.500      5.000      5.000 |            1.00x           10.00x           10.00x
small           512   8 |      4.000      4.000     88.000     88.000 |            1.00x           22.00x           22.00x
gpt2-ish       1024  12 |      6.000      6.000    228.000    228.000 |            1.00x           38.00x           38.00x
llama3 L=2k    2048  32 |     16.000     16.000    608.000    608.000 |            1.00x           38.00x           38.00x
llama3 L=4k    4096  32 |     32.000     32.000   2240.000   2240.000 |            1.00x           70.00x           70.00x
llama3 L=8k    8192  32 |     64.000     64.000        n/s        n/s |            1.00x                -                -

-- FULL GRADIENT, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      4.532      2.532     11.501     11.501 |            0.56x            2.54x            2.54x
small           512   8 |     36.250     20.250    188.002    188.002 |            0.56x            5.19x            5.19x
gpt2-ish       1024  12 |     54.375     30.375    474.001    474.001 |            0.56x            8.72x            8.72x
llama3 L=2k    2048  32 |    144.500     80.500   1264.001   1264.001 |            0.56x            8.75x            8.75x
llama3 L=4k    4096  32 |    289.000    161.000   4576.001   4576.001 |            0.56x           15.83x           15.83x
llama3 L=8k    8192  32 |    578.000    322.000        n/s        n/s |            0.56x                -                -

########################################################################################################
### dtype=Float16  causal=true
########################################################################################################

-- FORWARD (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.014      0.025      0.764      0.765 |            1.81x           54.99x           55.05x
small           512   8 |      0.037      0.144      1.486      1.487 |            3.86x           39.76x           39.78x
gpt2-ish       1024  12 |      0.080      0.344      1.532      1.525 |            4.32x           19.21x           19.12x
llama3 L=2k    2048  32 |      0.245      1.336      2.453      2.446 |            5.45x           10.01x            9.98x
llama3 L=4k    4096  32 |      0.798      4.683      7.834      7.821 |            5.86x            9.81x            9.80x
llama3 L=8k    8192  32 |      2.793     16.929        n/s        n/s |            6.06x                -                -

-- FULL GRADIENT, fwd+bwd (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.176      0.140      2.820      3.384 |            0.79x           16.02x           19.22x
small           512   8 |      0.219      0.784      5.617      6.150 |            3.58x           25.66x           28.10x
gpt2-ish       1024  12 |      0.329      1.823      7.405      7.990 |            5.55x           22.53x           24.30x
llama3 L=2k    2048  32 |      0.989      6.962     11.337     11.512 |            7.04x           11.46x           11.64x
llama3 L=4k    4096  32 |      3.155     23.281     38.110     37.885 |            7.38x           12.08x           12.01x
llama3 L=8k    8192  32 |     10.706     83.450        n/s        n/s |            7.79x                -                -

-- FORWARD, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.500      0.500      6.031      6.016 |            1.00x           12.06x           12.03x
small           512   8 |      4.000      4.000    120.500    120.250 |            1.00x           30.13x           30.06x
gpt2-ish       1024  12 |      6.000      6.000    326.000    325.000 |            1.00x           54.33x           54.17x
llama3 L=2k    2048  32 |     16.000     16.000    872.000    868.000 |            1.00x           54.50x           54.25x
llama3 L=4k    4096  32 |     32.000     32.000   3296.000   3280.000 |            1.00x          103.00x          102.50x
llama3 L=8k    8192  32 |     64.000     64.000        n/s        n/s |            1.00x                -                -

-- FULL GRADIENT, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      4.532      2.532     19.564     19.548 |            0.56x            4.32x            4.31x
small           512   8 |     36.250     20.250    445.002    444.752 |            0.56x           12.28x           12.27x
gpt2-ish       1024  12 |     54.375     30.375   1246.002   1245.002 |            0.56x           22.91x           22.90x
llama3 L=2k    2048  32 |    144.500     80.500   3328.001   3324.001 |            0.56x           23.03x           23.00x
llama3 L=4k    4096  32 |    289.000    161.000  12832.001  12816.001 |            0.56x           44.40x           44.35x
llama3 L=8k    8192  32 |    578.000    322.000        n/s        n/s |            0.56x                -                -

########################################################################################################
### dtype=BFloat16  causal=false
########################################################################################################

-- FORWARD (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.013      0.026      0.843      0.770 |            1.99x           65.59x           59.94x
small           512   8 |      0.036      0.205      1.501      1.503 |            5.62x           41.27x           41.33x
gpt2-ish       1024  12 |      0.118      0.509      1.226      1.230 |            4.33x           10.43x           10.46x
llama3 L=2k    2048  32 |      0.407      2.297      1.715      1.706 |            5.64x            4.21x            4.19x
llama3 L=4k    4096  32 |      1.388      8.448      4.572      4.582 |            6.09x            3.29x            3.30x
llama3 L=8k    8192  32 |      5.136     31.953        n/s        n/s |            6.22x                -                -

-- FULL GRADIENT, fwd+bwd (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.197      0.152      2.666      3.257 |            0.77x           13.56x           16.56x
small           512   8 |      0.207      1.051      4.993      5.507 |            5.07x           24.09x           26.57x
gpt2-ish       1024  12 |      0.461      2.574      4.245      4.742 |            5.58x            9.21x           10.29x
llama3 L=2k    2048  32 |      1.569     11.649      5.064      4.956 |            7.42x            3.23x            3.16x
llama3 L=4k    4096  32 |      5.398     41.707     12.700     12.435 |            7.73x            2.35x            2.30x
llama3 L=8k    8192  32 |     19.383    157.140        n/s        n/s |            8.11x                -                -

-- FORWARD, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.500      0.500      5.000      5.000 |            1.00x           10.00x           10.00x
small           512   8 |      4.000      4.000     88.001     88.001 |            1.00x           22.00x           22.00x
gpt2-ish       1024  12 |      6.000      6.000    228.001    228.001 |            1.00x           38.00x           38.00x
llama3 L=2k    2048  32 |     16.000     16.000    608.000    608.000 |            1.00x           38.00x           38.00x
llama3 L=4k    4096  32 |     32.000     32.000   2240.000   2240.000 |            1.00x           70.00x           70.00x
llama3 L=8k    8192  32 |     64.000     64.000        n/s        n/s |            1.00x                -                -

-- FULL GRADIENT, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      4.532      2.532     11.502     11.502 |            0.56x            2.54x            2.54x
small           512   8 |     36.250     20.250    188.004    188.004 |            0.56x            5.19x            5.19x
gpt2-ish       1024  12 |     54.375     30.375    474.003    474.003 |            0.56x            8.72x            8.72x
llama3 L=2k    2048  32 |    144.500     80.500   1264.002   1264.002 |            0.56x            8.75x            8.75x
llama3 L=4k    4096  32 |    289.000    161.000   4576.002   4576.002 |            0.56x           15.83x           15.83x
llama3 L=8k    8192  32 |    578.000    322.000        n/s        n/s |            0.56x                -                -

########################################################################################################
### dtype=BFloat16  causal=true
########################################################################################################

-- FORWARD (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.015      0.025      0.785      0.785 |            1.72x           54.05x           54.00x
small           512   8 |      0.037      0.143      1.672      1.521 |            3.84x           44.77x           40.74x
gpt2-ish       1024  12 |      0.080      0.342      1.604      1.605 |            4.26x           19.95x           19.96x
llama3 L=2k    2048  32 |      0.244      1.341      2.610      2.611 |            5.50x           10.70x           10.70x
llama3 L=4k    4096  32 |      0.790      4.679      8.240      8.211 |            5.92x           10.43x           10.40x
llama3 L=8k    8192  32 |      2.786     17.041        n/s        n/s |            6.12x                -                -

-- FULL GRADIENT, fwd+bwd (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.182      0.144      2.784      3.345 |            0.79x           15.30x           18.39x
small           512   8 |      0.208      0.784      6.170      6.193 |            3.76x           29.60x           29.71x
gpt2-ish       1024  12 |      0.332      1.810      7.409      8.315 |            5.45x           22.29x           25.02x
llama3 L=2k    2048  32 |      0.987      6.932     11.847     11.635 |            7.03x           12.01x           11.79x
llama3 L=4k    4096  32 |      3.149     23.251     38.940     38.684 |            7.38x           12.37x           12.28x
llama3 L=8k    8192  32 |     10.730     83.039        n/s        n/s |            7.74x                -                -

-- FORWARD, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.500      0.500      6.032      6.016 |            1.00x           12.06x           12.03x
small           512   8 |      4.000      4.000    120.501    120.251 |            1.00x           30.13x           30.06x
gpt2-ish       1024  12 |      6.000      6.000    326.001    325.001 |            1.00x           54.33x           54.17x
llama3 L=2k    2048  32 |     16.000     16.000    872.000    868.000 |            1.00x           54.50x           54.25x
llama3 L=4k    4096  32 |     32.000     32.000   3296.000   3280.000 |            1.00x          103.00x          102.50x
llama3 L=8k    8192  32 |     64.000     64.000        n/s        n/s |            1.00x                -                -

-- FULL GRADIENT, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      4.532      2.532     19.565     19.549 |            0.56x            4.32x            4.31x
small           512   8 |     36.250     20.250    445.004    444.754 |            0.56x           12.28x           12.27x
gpt2-ish       1024  12 |     54.375     30.375   1246.003   1245.003 |            0.56x           22.91x           22.90x
llama3 L=2k    2048  32 |    144.500     80.500   3328.002   3324.002 |            0.56x           23.03x           23.00x
llama3 L=4k    4096  32 |    289.000    161.000  12832.002  12816.002 |            0.56x           44.40x           44.35x
llama3 L=8k    8192  32 |    578.000    322.000        n/s        n/s |            0.56x                -                -

########################################################################################################
### dtype=Float32  causal=false
########################################################################################################

-- FORWARD (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.026      0.026      0.042      0.043 |            0.98x            1.60x            1.62x
small           512   8 |      0.205      0.206      0.246      0.244 |            1.00x            1.20x            1.19x
gpt2-ish       1024  12 |      0.511      0.510      0.747      0.751 |            1.00x            1.46x            1.47x
llama3 L=2k    2048  32 |      2.291      2.284      2.342      2.355 |            1.00x            1.02x            1.03x
llama3 L=4k    4096  32 |      8.216      8.244      8.753      8.750 |            1.00x            1.07x            1.07x
llama3 L=8k    8192  32 |     31.184     31.082        n/s        n/s |            1.00x                -                -

-- FULL GRADIENT, fwd+bwd (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.184      0.146      0.397      0.949 |            0.80x            2.16x            5.17x
small           512   8 |      1.054      1.054      0.963      1.108 |            1.00x            0.91x            1.05x
gpt2-ish       1024  12 |      2.587      2.604      2.228      2.122 |            1.01x            0.86x            0.82x
llama3 L=2k    2048  32 |     11.742     11.790      6.842      6.733 |            1.00x            0.58x            0.57x
llama3 L=4k    4096  32 |     42.295     42.191     24.839     24.740 |            1.00x            0.59x            0.58x
llama3 L=8k    8192  32 |    159.561    159.524        n/s        n/s |            1.00x                -                -

-- FORWARD, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      1.000      1.000     10.000     10.000 |            1.00x           10.00x           10.00x
small           512   8 |      8.000      8.000    176.000    176.000 |            1.00x           22.00x           22.00x
gpt2-ish       1024  12 |     12.000     12.000    456.000    456.000 |            1.00x           38.00x           38.00x
llama3 L=2k    2048  32 |     32.000     32.000   1216.000   1216.000 |            1.00x           38.00x           38.00x
llama3 L=4k    4096  32 |     64.000     64.000   4480.000   4480.000 |            1.00x           70.00x           70.00x
llama3 L=8k    8192  32 |    128.000    128.000        n/s        n/s |            1.00x                -                -

-- FULL GRADIENT, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      5.032      5.032     23.001     23.001 |            1.00x            4.57x            4.57x
small           512   8 |     40.251     40.251    376.001    376.001 |            1.00x            9.34x            9.34x
gpt2-ish       1024  12 |     60.376     60.376    948.001    948.001 |            1.00x           15.70x           15.70x
llama3 L=2k    2048  32 |    160.501    160.501   2528.001   2528.001 |            1.00x           15.75x           15.75x
llama3 L=4k    4096  32 |    321.001    321.001   9152.001   9152.001 |            1.00x           28.51x           28.51x
llama3 L=8k    8192  32 |    642.001    642.001        n/s        n/s |            1.00x                -                -

########################################################################################################
### dtype=Float32  causal=true
########################################################################################################

-- FORWARD (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.027      0.026      0.054      0.052 |            0.97x            2.02x            1.95x
small           512   8 |      0.143      0.142      0.376      0.372 |            1.00x            2.63x            2.60x
gpt2-ish       1024  12 |      0.339      0.344      1.225      1.222 |            1.01x            3.61x            3.60x
llama3 L=2k    2048  32 |      1.390      1.391      3.308      3.299 |            1.00x            2.38x            2.37x
llama3 L=4k    4096  32 |      4.597      4.617     12.557     12.499 |            1.00x            2.73x            2.72x
llama3 L=8k    8192  32 |     16.217     16.503        n/s        n/s |            1.02x                -                -

-- FULL GRADIENT, fwd+bwd (ms) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      0.179      0.148      0.451      1.067 |            0.83x            2.52x            5.96x
small           512   8 |      0.781      0.784      2.371      2.621 |            1.00x            3.04x            3.36x
gpt2-ish       1024  12 |      1.822      1.803      6.273      6.336 |            0.99x            3.44x            3.48x
llama3 L=2k    2048  32 |      6.998      7.011     17.413     17.234 |            1.00x            2.49x            2.46x
llama3 L=4k    4096  32 |     23.171     22.933     67.739     67.836 |            0.99x            2.92x            2.93x
llama3 L=8k    8192  32 |     82.346     82.588        OOM        OOM |            1.00x                -                -

-- FORWARD, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      1.000      1.000     12.031     12.016 |            1.00x           12.03x           12.02x
small           512   8 |      8.000      8.000    240.500    240.250 |            1.00x           30.06x           30.03x
gpt2-ish       1024  12 |     12.000     12.000    650.000    649.000 |            1.00x           54.17x           54.08x
llama3 L=2k    2048  32 |     32.000     32.000   1736.000   1732.000 |            1.00x           54.25x           54.13x
llama3 L=4k    4096  32 |     64.000     64.000   6560.000   6544.000 |            1.00x          102.50x          102.25x
llama3 L=8k    8192  32 |    128.000    128.000        n/s        n/s |            1.00x                -                -

-- FULL GRADIENT, GPU memory allocated (MiB) --
config            L   H |      nnlib   nnlib-ka  nnlib-mat        lux |   nnlib-ka/nnlib  nnlib-mat/nnlib        lux/nnlib
tiny            128   4 |      5.032      5.032     39.095     39.079 |            1.00x            7.77x            7.77x
small           512   8 |     40.251     40.251    889.501    889.251 |            1.00x           22.10x           22.09x
gpt2-ish       1024  12 |     60.376     60.376   2490.001   2489.001 |            1.00x           41.24x           41.23x
llama3 L=2k    2048  32 |    160.501    160.501   6648.001   6644.001 |            1.00x           41.42x           41.40x
llama3 L=4k    4096  32 |    321.001    321.001  25632.001  25616.001 |            1.00x           79.85x           79.80x
llama3 L=8k    8192  32 |    642.001    642.001        OOM        OOM |            1.00x                -                -

<impl>/nnlib = time relative to nnlib (>1 → nnlib faster).
