GPU:       NVIDIA GeForce RTX 5090
Julia:     1.13.1
NNlib:     0.9.45
CUDA.jl:   6.4.2
cuDNN.jl:  6.4.2 (library 9.26.0)
selection: ops=batchnorm,pooling,activations,softmax,conv  dtypes=Float16,Float32  sizes=small,medium,large,odd-C,long-W,long-C,long-WC
sanity on Float16 8x8x16x4: largest deviation from `nnlib`
  batchnorm (training)         generic=9.8e-04  fixed=0.0e+00  graph=2.4e-04
  batchnorm (inference)        generic=1.2e-04  fixed=0.0e+00  graph=n/s
  ∇batchnorm (training)        generic=0.0e+00  fixed=0.0e+00  graph=0.0e+00
  maxpool                      generic=-  fixed=0.0e+00  graph=0.0e+00
  ∇maxpool                     generic=-  fixed=0.0e+00  graph=0.0e+00
  meanpool                     generic=-  fixed=0.0e+00  graph=0.0e+00
  ∇meanpool                    generic=-  fixed=0.0e+00  graph=0.0e+00
  relu.(x)                     generic=0.0e+00  fixed=0.0e+00  graph=n/s
  tanh.(x)                     generic=0.0e+00  fixed=0.0e+00  graph=n/s
  σ.(x)                        generic=4.9e-04  fixed=0.0e+00  graph=n/s
  elu.(x)                      generic=4.9e-04  fixed=0.0e+00  graph=n/s
  softmax (dims=1)             generic=4.9e-04  fixed=0.0e+00  graph=-
  ∇softmax (dims=1)            generic=7.3e-04  fixed=0.0e+00  graph=-
  Zygote ∇softmax (dims=1)     generic=-  fixed=-  graph=-
  logsoftmax (dims=1)          generic=3.9e-03  fixed=0.0e+00  graph=-
  ∇logsoftmax (dims=1)         generic=3.9e-03  fixed=0.0e+00  graph=-
  Zygote ∇logsoftmax (dims=1)  generic=-  fixed=-  graph=-
  softmax (dims=3)             generic=0.0e+00  fixed=4.9e-04  graph=-
  ∇softmax (dims=3)            generic=0.0e+00  fixed=4.9e-04  graph=-
  Zygote ∇softmax (dims=3)     generic=-  fixed=-  graph=-
  logsoftmax (dims=3)          generic=0.0e+00  fixed=3.9e-03  graph=-
  ∇logsoftmax (dims=3)         generic=0.0e+00  fixed=3.9e-03  graph=-
  Zygote ∇logsoftmax (dims=3)  generic=-  fixed=-  graph=-
  conv                         generic=-  fixed=0.0e+00  graph=0.0e+00
  ∇conv_data                   generic=-  fixed=9.8e-04  graph=0.0e+00
  ∇conv_filter                 generic=-  fixed=0.0e+00  graph=0.0e+00

########################################################################################################
### batchnorm   (μs)
########################################################################################################
operator                     eltype    size           |     nnlib   generic     fixed     graph | fastest
batchnorm (training)         Float16   8x8x16x4       |       7.5      31.5       7.4      22.1 | fixed
batchnorm (inference)        Float16   8x8x16x4       |       6.1      18.4       5.9       n/s | fixed
∇batchnorm (training)        Float16   8x8x16x4       |       9.1      52.3       8.1      21.2 | fixed
batchnorm (training)         Float16   32x32x64x16    |       9.1      78.7       8.9      24.0 | fixed
batchnorm (inference)        Float16   32x32x64x16    |       7.7      44.8       7.9       n/s | fixed
∇batchnorm (training)        Float16   32x32x64x16    |      19.0     146.8      11.3      24.3 | fixed
batchnorm (training)         Float16   64x64x128x32   |      57.2     710.6      57.4      92.6 | fixed
batchnorm (inference)        Float16   64x64x128x32   |      22.7     474.8      22.7       n/s | fixed
∇batchnorm (training)        Float16   64x64x128x32   |     101.4    1449.9      85.8     154.0 | fixed
batchnorm (training)         Float16   32x32x30x16    |       8.9      52.4       8.8       n/s | fixed
batchnorm (inference)        Float16   32x32x30x16    |       6.8      27.5       6.9       n/s | fixed
∇batchnorm (training)        Float16   32x32x30x16    |      20.8      95.1      11.3       n/s | fixed
batchnorm (training)         Float32   8x8x16x4       |       7.5      25.3       7.5      20.1 | fixed
batchnorm (inference)        Float32   8x8x16x4       |       6.1      11.9       5.9       n/s | fixed
∇batchnorm (training)        Float32   8x8x16x4       |       8.9      44.9       8.6      21.4 | fixed
batchnorm (training)         Float32   32x32x64x16    |      12.0      59.8      12.1      26.1 | fixed
batchnorm (inference)        Float32   32x32x64x16    |       8.2      25.4       8.1       n/s | fixed
∇batchnorm (training)        Float32   32x32x64x16    |      19.0     117.4      16.4      27.4 | fixed
batchnorm (training)         Float32   64x64x128x32   |     114.5     467.2     116.1     215.8 | fixed
batchnorm (inference)        Float32   64x64x128x32   |      93.9     220.2      93.8       n/s | fixed
∇batchnorm (training)        Float32   64x64x128x32   |     268.3    1066.0     242.8     354.9 | fixed
batchnorm (training)         Float32   32x32x30x16    |      12.5      45.0      12.0       n/s | fixed
batchnorm (inference)        Float32   32x32x30x16    |       7.3      18.3       7.4       n/s | fixed
∇batchnorm (training)        Float32   32x32x30x16    |      18.4      84.7      16.4       n/s | fixed

########################################################################################################
### pooling   (μs)
########################################################################################################
operator                     eltype    size           |     nnlib   generic     fixed     graph | fastest
maxpool                      Float16   8x8x16x4       |       7.2         -       6.3       7.5 | fixed
∇maxpool                     Float16   8x8x16x4       |       8.0         -       7.0       9.3 | fixed
meanpool                     Float16   8x8x16x4       |       7.9         -       6.5       7.7 | fixed
∇meanpool                    Float16   8x8x16x4       |       8.2         -       7.0       8.5 | fixed
maxpool                      Float16   32x32x64x16    |       8.5         -       7.1       8.3 | fixed
∇maxpool                     Float16   32x32x64x16    |      10.0         -       8.7      13.1 | fixed
meanpool                     Float16   32x32x64x16    |       8.6         -       7.4       8.4 | fixed
∇meanpool                    Float16   32x32x64x16    |       9.9         -       8.5      10.1 | fixed
maxpool                      Float16   64x64x128x32   |      19.3         -      17.3      19.4 | fixed
∇maxpool                     Float16   64x64x128x32   |      52.8         -      33.7      87.1 | fixed
meanpool                     Float16   64x64x128x32   |      20.7         -      18.6      19.8 | fixed
∇meanpool                    Float16   64x64x128x32   |      47.0         -      36.7      38.1 | fixed
maxpool                      Float16   32x32x30x16    |       8.2         -       6.8       7.8 | fixed
∇maxpool                     Float16   32x32x30x16    |       9.0         -       7.7      10.4 | fixed
meanpool                     Float16   32x32x30x16    |       8.8         -       7.4       8.7 | fixed
∇meanpool                    Float16   32x32x30x16    |       9.0         -       7.7       9.3 | fixed
maxpool                      Float32   8x8x16x4       |       7.4         -       6.5       7.6 | fixed
∇maxpool                     Float32   8x8x16x4       |       7.9         -       6.6       8.8 | fixed
meanpool                     Float32   8x8x16x4       |       7.5         -       6.6       7.8 | fixed
∇meanpool                    Float32   8x8x16x4       |       8.1         -       6.5       8.4 | fixed
maxpool                      Float32   32x32x64x16    |       8.6         -       7.3       8.4 | fixed
∇maxpool                     Float32   32x32x64x16    |      10.6         -       9.2      13.0 | fixed
meanpool                     Float32   32x32x64x16    |       8.7         -       7.4       8.3 | fixed
∇meanpool                    Float32   32x32x64x16    |      10.0         -       8.6      10.2 | fixed
maxpool                      Float32   64x64x128x32   |      50.4         -      18.5      19.8 | fixed
∇maxpool                     Float32   64x64x128x32   |     143.8         -     119.0     129.0 | fixed
meanpool                     Float32   64x64x128x32   |      51.1         -      18.4      19.6 | fixed
∇meanpool                    Float32   64x64x128x32   |      64.9         -      39.5      39.7 | fixed
maxpool                      Float32   32x32x30x16    |       8.2         -       6.9       8.0 | fixed
∇maxpool                     Float32   32x32x30x16    |       9.1         -       7.8      10.5 | fixed
meanpool                     Float32   32x32x30x16    |       8.3         -       7.0       8.0 | fixed
∇meanpool                    Float32   32x32x30x16    |       9.0         -       8.1      10.0 | fixed

########################################################################################################
### activations   (μs)
########################################################################################################
operator                     eltype    size           |     nnlib   generic     fixed     graph | fastest
relu.(x)                     Float16   8x8x16x4       |       5.7       7.0       5.9       n/s | fixed
tanh.(x)                     Float16   8x8x16x4       |       5.6       7.0       5.9       n/s | fixed
σ.(x)                        Float16   8x8x16x4       |       5.7       7.0       5.9       n/s | fixed
elu.(x)                      Float16   8x8x16x4       |       5.7       7.0       5.9       n/s | fixed
relu.(x)                     Float16   32x32x64x16    |       7.0      15.9       7.2       n/s | fixed
tanh.(x)                     Float16   32x32x64x16    |       7.3      16.3       7.4       n/s | fixed
σ.(x)                        Float16   32x32x64x16    |       7.5      16.8       7.6       n/s | fixed
elu.(x)                      Float16   32x32x64x16    |       7.4      16.3       7.4       n/s | fixed
relu.(x)                     Float16   64x64x128x32   |      22.6     129.1      23.0       n/s | fixed
tanh.(x)                     Float16   64x64x128x32   |      26.0     135.1      26.0       n/s | fixed
σ.(x)                        Float16   64x64x128x32   |      26.9     141.6      27.0       n/s | fixed
elu.(x)                      Float16   64x64x128x32   |      25.5     137.1      25.8       n/s | fixed
relu.(x)                     Float16   32x32x30x16    |       6.4      10.8       6.5       n/s | fixed
tanh.(x)                     Float16   32x32x30x16    |       6.5      11.1       6.6       n/s | fixed
σ.(x)                        Float16   32x32x30x16    |       6.8      11.4       6.8       n/s | fixed
elu.(x)                      Float16   32x32x30x16    |       6.5      11.1       6.6       n/s | fixed
relu.(x)                     Float32   8x8x16x4       |       5.6       7.1       5.8       n/s | fixed
tanh.(x)                     Float32   8x8x16x4       |       5.7       7.2       6.2       n/s | fixed
σ.(x)                        Float32   8x8x16x4       |       6.0       7.4       6.2       n/s | fixed
elu.(x)                      Float32   8x8x16x4       |       5.7       6.9       5.9       n/s | fixed
relu.(x)                     Float32   32x32x64x16    |       7.5      15.9       7.7       n/s | fixed
tanh.(x)                     Float32   32x32x64x16    |       7.8      16.2       7.9       n/s | fixed
σ.(x)                        Float32   32x32x64x16    |       7.8      16.3       8.1       n/s | fixed
elu.(x)                      Float32   32x32x64x16    |       7.8      15.9       7.9       n/s | fixed
relu.(x)                     Float32   64x64x128x32   |      90.3     138.8      91.4       n/s | fixed
tanh.(x)                     Float32   64x64x128x32   |      90.7     142.4      90.8       n/s | fixed
σ.(x)                        Float32   64x64x128x32   |      90.6     143.7      90.6       n/s | fixed
elu.(x)                      Float32   64x64x128x32   |      91.0     139.2      91.1       n/s | fixed
relu.(x)                     Float32   32x32x30x16    |       6.8      10.8       6.7       n/s | fixed
tanh.(x)                     Float32   32x32x30x16    |       6.8      11.1       6.8       n/s | fixed
σ.(x)                        Float32   32x32x30x16    |       6.7      11.2       6.8       n/s | fixed
elu.(x)                      Float32   32x32x30x16    |       6.7      10.9       6.7       n/s | fixed

########################################################################################################
### softmax   (μs)
########################################################################################################
operator                     eltype    size           |     nnlib   generic     fixed     graph | fastest
softmax (dims=1)             Float16   8x8x16x4       |       5.5      30.4       5.6         - | fixed
∇softmax (dims=1)            Float16   8x8x16x4       |       5.6      14.1       5.5         - | fixed
Zygote ∇softmax (dims=1)     Float16   8x8x16x4       |       5.5         -         -         - | -
logsoftmax (dims=1)          Float16   8x8x16x4       |       5.4      33.0       5.6         - | fixed
∇logsoftmax (dims=1)         Float16   8x8x16x4       |       5.5      11.4       5.6         - | fixed
Zygote ∇logsoftmax (dims=1)  Float16   8x8x16x4       |       5.6         -         -         - | -
softmax (dims=3)             Float16   8x8x16x4       |      30.1      30.6       6.8         - | fixed
∇softmax (dims=3)            Float16   8x8x16x4       |      14.3      14.3       6.7         - | fixed
Zygote ∇softmax (dims=3)     Float16   8x8x16x4       |      14.3         -         -         - | -
logsoftmax (dims=3)          Float16   8x8x16x4       |      33.1      33.3       6.6         - | fixed
∇logsoftmax (dims=3)         Float16   8x8x16x4       |      11.3      11.5       8.6         - | fixed
Zygote ∇logsoftmax (dims=3)  Float16   8x8x16x4       |      11.5         -         -         - | -
softmax (dims=1)             Float16   32x32x64x16    |       7.9      85.6       7.8         - | fixed
∇softmax (dims=1)            Float16   32x32x64x16    |       7.6      49.5       7.8         - | fixed
Zygote ∇softmax (dims=1)     Float16   32x32x64x16    |       7.8         -         -         - | -
logsoftmax (dims=1)          Float16   32x32x64x16    |       7.7      85.2       7.9         - | fixed
∇logsoftmax (dims=1)         Float16   32x32x64x16    |       7.7      38.2       7.8         - | fixed
Zygote ∇logsoftmax (dims=1)  Float16   32x32x64x16    |       7.8         -         -         - | -
softmax (dims=3)             Float16   32x32x64x16    |      83.1      83.3      11.2         - | fixed
∇softmax (dims=3)            Float16   32x32x64x16    |      48.8      48.9       9.8         - | fixed
Zygote ∇softmax (dims=3)     Float16   32x32x64x16    |      49.0         -         -         - | -
logsoftmax (dims=3)          Float16   32x32x64x16    |      84.3      84.4       9.9         - | fixed
∇logsoftmax (dims=3)         Float16   32x32x64x16    |      38.3      38.3      18.0         - | fixed
Zygote ∇logsoftmax (dims=3)  Float16   32x32x64x16    |      38.1         -         -         - | -
softmax (dims=1)             Float16   64x64x128x32   |      25.3     592.6      25.6         - | fixed
∇softmax (dims=1)            Float16   64x64x128x32   |      53.6     458.5      52.9         - | fixed
Zygote ∇softmax (dims=1)     Float16   64x64x128x32   |      53.2         -         -         - | -
logsoftmax (dims=1)          Float16   64x64x128x32   |      25.9     598.7      26.1         - | fixed
∇logsoftmax (dims=1)         Float16   64x64x128x32   |      53.2     319.7      55.4         - | fixed
Zygote ∇logsoftmax (dims=1)  Float16   64x64x128x32   |      52.8         -         -         - | -
softmax (dims=3)             Float16   64x64x128x32   |     802.1     800.0      53.3         - | fixed
∇softmax (dims=3)            Float16   64x64x128x32   |     559.6     560.0      56.7         - | fixed
Zygote ∇softmax (dims=3)     Float16   64x64x128x32   |     559.7         -         -         - | -
logsoftmax (dims=3)          Float16   64x64x128x32   |     806.2     804.8      51.1         - | fixed
∇logsoftmax (dims=3)         Float16   64x64x128x32   |     427.6     427.4      65.8         - | fixed
Zygote ∇logsoftmax (dims=3)  Float16   64x64x128x32   |     427.4         -         -         - | -
softmax (dims=1)             Float16   32x32x30x16    |       6.8      55.9       7.0         - | fixed
∇softmax (dims=1)            Float16   32x32x30x16    |       6.9      29.0       6.7         - | fixed
Zygote ∇softmax (dims=1)     Float16   32x32x30x16    |       6.6         -         -         - | -
logsoftmax (dims=1)          Float16   32x32x30x16    |       6.5      56.7       6.8         - | fixed
∇logsoftmax (dims=1)         Float16   32x32x30x16    |       6.7      23.4       6.8         - | fixed
Zygote ∇logsoftmax (dims=1)  Float16   32x32x30x16    |       6.7         -         -         - | -
softmax (dims=3)             Float16   32x32x30x16    |      58.3      58.1       8.8         - | fixed
∇softmax (dims=3)            Float16   32x32x30x16    |      30.1      30.1       8.1         - | fixed
Zygote ∇softmax (dims=3)     Float16   32x32x30x16    |      30.2         -         -         - | -
logsoftmax (dims=3)          Float16   32x32x30x16    |      58.8      58.8       8.3         - | fixed
∇logsoftmax (dims=3)         Float16   32x32x30x16    |      24.6      24.6      11.9         - | fixed
Zygote ∇logsoftmax (dims=3)  Float16   32x32x30x16    |      24.5         -         -         - | -
softmax (dims=1)             Float16   10000x1x100x1  |       9.6      71.4       9.6         - | fixed
∇softmax (dims=1)            Float16   10000x1x100x1  |       9.0      38.8       9.4         - | fixed
Zygote ∇softmax (dims=1)     Float16   10000x1x100x1  |       9.1         -         -         - | -
logsoftmax (dims=1)          Float16   10000x1x100x1  |       9.1      72.1       9.5         - | fixed
∇logsoftmax (dims=1)         Float16   10000x1x100x1  |      14.4      33.7      14.5         - | fixed
Zygote ∇logsoftmax (dims=1)  Float16   10000x1x100x1  |      14.4         -         -         - | -
softmax (dims=3)             Float16   10000x1x100x1  |      81.4      81.8      21.8         - | fixed
∇softmax (dims=3)            Float16   10000x1x100x1  |      44.2      44.3      23.9         - | fixed
Zygote ∇softmax (dims=3)     Float16   10000x1x100x1  |      44.1         -         -         - | -
logsoftmax (dims=3)          Float16   10000x1x100x1  |      81.8      82.2      20.2         - | fixed
∇logsoftmax (dims=3)         Float16   10000x1x100x1  |      36.8      36.7      25.2         - | fixed
Zygote ∇logsoftmax (dims=3)  Float16   10000x1x100x1  |      36.8         -         -         - | -
softmax (dims=1)             Float16   100x1x10000x1  |       6.8      79.7       7.2         - | fixed
∇softmax (dims=1)            Float16   100x1x10000x1  |       7.2      44.1       7.1         - | fixed
Zygote ∇softmax (dims=1)     Float16   100x1x10000x1  |       7.0         -         -         - | -
logsoftmax (dims=1)          Float16   100x1x10000x1  |       7.0      81.3       7.1         - | fixed
∇logsoftmax (dims=1)         Float16   100x1x10000x1  |       7.3      36.0       7.2         - | fixed
Zygote ∇logsoftmax (dims=1)  Float16   100x1x10000x1  |       7.3         -         -         - | -
softmax (dims=3)             Float16   100x1x10000x1  |      85.7      85.5      78.3         - | fixed
∇softmax (dims=3)            Float16   100x1x10000x1  |      44.9      45.0    1820.7         - | generic
Zygote ∇softmax (dims=3)     Float16   100x1x10000x1  |      45.7         -         -         - | -
logsoftmax (dims=3)          Float16   100x1x10000x1  |      85.4      84.3      52.0         - | fixed
∇logsoftmax (dims=3)         Float16   100x1x10000x1  |      40.3      38.7    1851.8         - | generic
Zygote ∇logsoftmax (dims=3)  Float16   100x1x10000x1  |      40.8         -         -         - | -
softmax (dims=1)             Float16   1000x1x1000x1  |       8.8      81.3       9.0         - | fixed
∇softmax (dims=1)            Float16   1000x1x1000x1  |       8.2      45.4       8.2         - | fixed
Zygote ∇softmax (dims=1)     Float16   1000x1x1000x1  |       8.2         -         -         - | -
logsoftmax (dims=1)          Float16   1000x1x1000x1  |       8.6      80.5       8.9         - | fixed
∇logsoftmax (dims=1)         Float16   1000x1x1000x1  |       8.2      36.4       8.3         - | fixed
Zygote ∇logsoftmax (dims=1)  Float16   1000x1x1000x1  |       8.4         -         -         - | -
softmax (dims=3)             Float16   1000x1x1000x1  |      94.8      92.7     332.0         - | generic
∇softmax (dims=3)            Float16   1000x1x1000x1  |      52.1      51.8     194.2         - | generic
Zygote ∇softmax (dims=3)     Float16   1000x1x1000x1  |      52.1         -         -         - | -
logsoftmax (dims=3)          Float16   1000x1x1000x1  |      94.8      94.1     314.3         - | generic
∇logsoftmax (dims=3)         Float16   1000x1x1000x1  |      43.0      43.0     197.8         - | generic
Zygote ∇logsoftmax (dims=3)  Float16   1000x1x1000x1  |      43.3         -         -         - | -
softmax (dims=1)             Float32   8x8x16x4       |       5.7      34.2       5.6         - | fixed
∇softmax (dims=1)            Float32   8x8x16x4       |       5.6      15.9       6.0         - | fixed
Zygote ∇softmax (dims=1)     Float32   8x8x16x4       |       5.8         -         -         - | -
logsoftmax (dims=1)          Float32   8x8x16x4       |       5.4      34.9       5.8         - | fixed
∇logsoftmax (dims=1)         Float32   8x8x16x4       |       5.6      11.8       5.8         - | fixed
Zygote ∇logsoftmax (dims=1)  Float32   8x8x16x4       |       5.6         -         -         - | -
softmax (dims=3)             Float32   8x8x16x4       |      33.5      33.3       6.9         - | fixed
∇softmax (dims=3)            Float32   8x8x16x4       |      14.5      14.5       6.6         - | fixed
Zygote ∇softmax (dims=3)     Float32   8x8x16x4       |      14.8         -         -         - | -
logsoftmax (dims=3)          Float32   8x8x16x4       |      35.0      35.4       6.6         - | fixed
∇logsoftmax (dims=3)         Float32   8x8x16x4       |      12.2      12.2       8.8         - | fixed
Zygote ∇logsoftmax (dims=3)  Float32   8x8x16x4       |      11.8         -         -         - | -
softmax (dims=1)             Float32   32x32x64x16    |       7.7      84.3       7.9         - | fixed
∇softmax (dims=1)            Float32   32x32x64x16    |       7.8      49.4       8.0         - | fixed
Zygote ∇softmax (dims=1)     Float32   32x32x64x16    |       7.9         -         -         - | -
logsoftmax (dims=1)          Float32   32x32x64x16    |       7.7      85.1       7.9         - | fixed
∇logsoftmax (dims=1)         Float32   32x32x64x16    |       8.0      37.6       8.0         - | fixed
Zygote ∇logsoftmax (dims=1)  Float32   32x32x64x16    |       8.0         -         -         - | -
softmax (dims=3)             Float32   32x32x64x16    |      82.2      83.1      20.6         - | fixed
∇softmax (dims=3)            Float32   32x32x64x16    |      48.5      48.8      18.0         - | fixed
Zygote ∇softmax (dims=3)     Float32   32x32x64x16    |      48.7         -         -         - | -
logsoftmax (dims=3)          Float32   32x32x64x16    |      83.1      83.7      19.7         - | fixed
∇logsoftmax (dims=3)         Float32   32x32x64x16    |      37.5      37.0      18.0         - | fixed
Zygote ∇logsoftmax (dims=3)  Float32   32x32x64x16    |      37.3         -         -         - | -
softmax (dims=1)             Float32   64x64x128x32   |      89.1     621.7      75.3         - | fixed
∇softmax (dims=1)            Float32   64x64x128x32   |     126.4     481.9     128.0         - | fixed
Zygote ∇softmax (dims=1)     Float32   64x64x128x32   |     130.4         -         -         - | -
logsoftmax (dims=1)          Float32   64x64x128x32   |      75.7     631.8      90.2         - | fixed
∇logsoftmax (dims=1)         Float32   64x64x128x32   |     127.0     334.1     130.6         - | fixed
Zygote ∇logsoftmax (dims=1)  Float32   64x64x128x32   |     129.7         -         -         - | -
softmax (dims=3)             Float32   64x64x128x32   |     834.2     833.2     167.9         - | fixed
∇softmax (dims=3)            Float32   64x64x128x32   |     588.1     588.5     216.9         - | fixed
Zygote ∇softmax (dims=3)     Float32   64x64x128x32   |     588.2         -         -         - | -
logsoftmax (dims=3)          Float32   64x64x128x32   |     834.4     834.1     168.8         - | fixed
∇logsoftmax (dims=3)         Float32   64x64x128x32   |     439.1     440.7     174.5         - | fixed
Zygote ∇logsoftmax (dims=3)  Float32   64x64x128x32   |     439.6         -         -         - | -
softmax (dims=1)             Float32   32x32x30x16    |       7.0      59.1       6.9         - | fixed
∇softmax (dims=1)            Float32   32x32x30x16    |       6.7      29.1       6.7         - | fixed
Zygote ∇softmax (dims=1)     Float32   32x32x30x16    |       6.7         -         -         - | -
logsoftmax (dims=1)          Float32   32x32x30x16    |       6.6      57.6       6.7         - | fixed
∇logsoftmax (dims=1)         Float32   32x32x30x16    |       6.7      23.1       6.7         - | fixed
Zygote ∇logsoftmax (dims=1)  Float32   32x32x30x16    |       6.6         -         -         - | -
softmax (dims=3)             Float32   32x32x30x16    |      60.4      59.0       8.8         - | fixed
∇softmax (dims=3)            Float32   32x32x30x16    |      30.0      29.8      10.8         - | fixed
Zygote ∇softmax (dims=3)     Float32   32x32x30x16    |      29.9         -         -         - | -
logsoftmax (dims=3)          Float32   32x32x30x16    |      60.8      61.1       8.7         - | fixed
∇logsoftmax (dims=3)         Float32   32x32x30x16    |      23.9      24.1      12.0         - | fixed
Zygote ∇logsoftmax (dims=3)  Float32   32x32x30x16    |      23.8         -         -         - | -
softmax (dims=1)             Float32   10000x1x100x1  |       8.1      80.5       8.2         - | fixed
∇softmax (dims=1)            Float32   10000x1x100x1  |      13.0      39.0      13.0         - | fixed
Zygote ∇softmax (dims=1)     Float32   10000x1x100x1  |      12.9         -         -         - | -
logsoftmax (dims=1)          Float32   10000x1x100x1  |       8.4      85.7       8.7         - | fixed
∇logsoftmax (dims=1)         Float32   10000x1x100x1  |      14.3      33.9      14.3         - | fixed
Zygote ∇logsoftmax (dims=1)  Float32   10000x1x100x1  |      14.2         -         -         - | -
softmax (dims=3)             Float32   10000x1x100x1  |      79.8      82.0      35.9         - | fixed
∇softmax (dims=3)            Float32   10000x1x100x1  |      43.9      43.8      24.4         - | fixed
Zygote ∇softmax (dims=3)     Float32   10000x1x100x1  |      43.8         -         -         - | -
logsoftmax (dims=3)          Float32   10000x1x100x1  |      82.1      80.9      34.5         - | fixed
∇logsoftmax (dims=3)         Float32   10000x1x100x1  |      36.3      36.1      24.6         - | fixed
Zygote ∇logsoftmax (dims=3)  Float32   10000x1x100x1  |      35.8         -         -         - | -
softmax (dims=1)             Float32   100x1x10000x1  |       7.2      79.5       7.4         - | fixed
∇softmax (dims=1)            Float32   100x1x10000x1  |       7.8      44.4       8.2         - | fixed
Zygote ∇softmax (dims=1)     Float32   100x1x10000x1  |       7.9         -         -         - | -
logsoftmax (dims=1)          Float32   100x1x10000x1  |       7.2      82.3       7.3         - | fixed
∇logsoftmax (dims=1)         Float32   100x1x10000x1  |       8.0      35.3       8.1         - | fixed
Zygote ∇logsoftmax (dims=1)  Float32   100x1x10000x1  |       7.9         -         -         - | -
softmax (dims=3)             Float32   100x1x10000x1  |      87.6      83.9      65.8         - | fixed
∇softmax (dims=3)            Float32   100x1x10000x1  |      45.9      45.6    1753.1         - | generic
Zygote ∇softmax (dims=3)     Float32   100x1x10000x1  |      45.8         -         -         - | -
logsoftmax (dims=3)          Float32   100x1x10000x1  |      84.7      89.6      46.3         - | fixed
∇logsoftmax (dims=3)         Float32   100x1x10000x1  |      40.6      39.5    1770.1         - | generic
Zygote ∇logsoftmax (dims=3)  Float32   100x1x10000x1  |      40.1         -         -         - | -
softmax (dims=1)             Float32   1000x1x1000x1  |       8.6      80.3       8.8         - | fixed
∇softmax (dims=1)            Float32   1000x1x1000x1  |       9.4      45.2       9.4         - | fixed
Zygote ∇softmax (dims=1)     Float32   1000x1x1000x1  |       9.4         -         -         - | -
logsoftmax (dims=1)          Float32   1000x1x1000x1  |       8.6      82.5       8.7         - | fixed
∇logsoftmax (dims=1)         Float32   1000x1x1000x1  |       9.3      35.7       9.3         - | fixed
Zygote ∇logsoftmax (dims=1)  Float32   1000x1x1000x1  |       9.3         -         -         - | -
softmax (dims=3)             Float32   1000x1x1000x1  |      92.6      95.7     323.1         - | generic
∇softmax (dims=3)            Float32   1000x1x1000x1  |      51.9      51.8     188.3         - | generic
Zygote ∇softmax (dims=3)     Float32   1000x1x1000x1  |      51.7         -         -         - | -
logsoftmax (dims=3)          Float32   1000x1x1000x1  |      94.8      92.8     308.9         - | generic
∇logsoftmax (dims=3)         Float32   1000x1x1000x1  |      42.5      42.4     190.8         - | generic
Zygote ∇logsoftmax (dims=3)  Float32   1000x1x1000x1  |      42.4         -         -         - | -

########################################################################################################
### conv   (μs)
########################################################################################################
operator                     eltype    size           |     nnlib   generic     fixed     graph | fastest
conv                         Float16   8x8x16x4       |      15.6         -      28.8      15.5 | graph
∇conv_data                   Float16   8x8x16x4       |      17.2         -      27.2      16.9 | graph
∇conv_filter                 Float16   8x8x16x4       |       8.3         -      18.1       8.0 | graph
conv                         Float16   32x32x64x16    |      24.6         -      44.0      25.2 | graph
∇conv_data                   Float16   32x32x64x16    |      24.5         -      38.8      24.1 | graph
∇conv_filter                 Float16   32x32x64x16    |      30.5         -      44.8      30.3 | graph
conv                         Float16   64x64x128x32   |     292.6         -     315.7     292.9 | graph
∇conv_data                   Float16   64x64x128x32   |     261.2         -     279.7     260.8 | graph
∇conv_filter                 Float16   64x64x128x32   |     393.2         -     412.5     392.8 | graph
conv                         Float16   32x32x30x16    |      26.5         -      34.7      25.2 | graph
∇conv_data                   Float16   32x32x30x16    |      29.9         -      34.5      29.3 | graph
∇conv_filter                 Float16   32x32x30x16    |      28.1         -      42.9      28.0 | graph
conv                         Float32   8x8x16x4       |      17.6         -      22.4      16.6 | graph
∇conv_data                   Float32   8x8x16x4       |      12.2         -      20.0      11.9 | graph
∇conv_filter                 Float32   8x8x16x4       |      12.2         -      20.0      11.9 | graph
conv                         Float32   32x32x64x16    |      32.4         -      41.0      32.2 | graph
∇conv_data                   Float32   32x32x64x16    |      32.3         -      39.9      32.1 | graph
∇conv_filter                 Float32   32x32x64x16    |      49.1         -      61.1      49.0 | graph
conv                         Float32   64x64x128x32   |     491.3         -     512.5     491.6 | graph
∇conv_data                   Float32   64x64x128x32   |     547.2         -     505.5     548.0 | fixed
∇conv_filter                 Float32   64x64x128x32   |     676.8         -     581.2     676.2 | fixed
conv                         Float32   32x32x30x16    |      31.4         -      29.8      30.2 | fixed
∇conv_data                   Float32   32x32x30x16    |      30.6         -      32.7      30.4 | graph
∇conv_filter                 Float32   32x32x30x16    |      38.9         -      35.2      38.4 | fixed
┌ Warning: unsupported path → n/s: cuDNN: cannot build graph NormFwdOp (CUDNN_STATUS_NOT_SUPPORTED)
└ @ Main ~/.julia/dev/NNlib/benchmark/cudnn/bench_cudnn.jl:426
┌ Warning: unsupported path → n/s: cuDNN: no supported engine for graph NormFwdOp (0 candidate configs)
└ @ Main ~/.julia/dev/NNlib/benchmark/cudnn/bench_cudnn.jl:426
┌ Warning: unsupported path → n/s: cuDNN: no supported engine for graph PointwiseOp (0 candidate configs)
└ @ Main ~/.julia/dev/NNlib/benchmark/cudnn/bench_cudnn.jl:426

fastest = the fastest of generic, fixed and graph; `nnlib` is what NNlib runs.
