Introduction to Deep Learning & Generative AI (BSDA2001P) formula sheet

$\hat{y} = w_1 \cdot x_1 + w_2 \cdot x_2 + b$

$C_{ij} = \sum_{k=1}^{n} A_{ik} \cdot B_{kj}$

$\texttt{torch.manual\_seed}(s) \quad \text{and} \quad \texttt{torch.cuda.manual\_seed}(s)$

$y = w \cdot x + b$

$\texttt{self.weight} = \texttt{nn.Parameter}(\texttt{torch.randn}(1))$

$\mathcal{L}_{\text{MAE}} = \frac{1}{N} \sum_{i=1}^{N} |\hat{y}_i - y_i|$

$\texttt{torch.save}(\texttt{model.state\_dict}(),\, \texttt{path})$

$\text{Sum} = x_1 + x_2 + x_3 = 1 + 0 + 1 = 2$

$\text{output} = \begin{cases} 1 & \text{if } \sum_{i} x_i \geq \theta \\ 0 & \text{otherwise} \end{cases}$

$z = w_1 x_1 + w_2 x_2 = 1 \times 2 + 1 \times (-1) = 2 - 1 = 1$

$\hat{y} = \begin{cases} 1 & \text{if } \sum_{i} w_i x_i \geq \theta \\ 0 & \text{otherwise} \end{cases}$

$z = w_1 x_1 + w_2 x_2 + b = 0.5 \times 1 + (-1) \times 2 + 0.1 = 0.5 - 2 + 0.1 = -1.4$

$a = g(z) = \frac{1}{1 + e^{-z}} = \frac{1}{1 + e^{1.4}} \approx \frac{1}{1 + 4.055} \approx \frac{1}{5.055} \approx 0.198$

$z = \sum_{i} w_i x_i + b, \quad a = g(z)$

$z = w_1 x_1 + w_2 x_2 + b = 0 \times 1 + 0 \times 1 + 0 = 0$

$a = \frac{1}{1 + e^{-z}} = \frac{1}{1 + e^{0}} = \frac{1}{2} = 0.5$

$\text{pedestrian} \rightarrow [1, 0, 0], \quad \text{vehicle} \rightarrow [0, 1, 0], \quad \text{stop sign} \rightarrow [0, 0, 1]$

$\hat{y} = f_{\theta}(x)$

$\theta^* = \arg\min_{\theta} \; L(\theta)$

$z = w_1 x_1 + w_2 x_2 + b$

$z = 2 \times 0.5 + 7 \times 1.0 + 0.1$

$z = 1.0 + 7.0 + 0.1$

$z = 8.1$

$a = g(z) = \text{ReLU}(8.1) = \max(0,\, 8.1) = 8.1$

$z = W x + b = \begin{bmatrix} 2 & 7 \\ 3 & 4 \\ 1 & 2 \end{bmatrix} \begin{bmatrix} 0.5 \\ 1.0 \end{bmatrix} + \begin{bmatrix} 0.1 \\ 2 \\ -0.5 \end{bmatrix}$

$W x = \begin{bmatrix} 2(0.5) + 7(1.0) \\ 3(0.5) + 4(1.0) \\ 1(0.5) + 2(1.0) \end{bmatrix} = \begin{bmatrix} 1.0 + 7.0 \\ 1.5 + 4.0 \\ 0.5 + 2.0 \end{bmatrix} = \begin{bmatrix} 8.0 \\ 5.5 \\ 2.5 \end{bmatrix}$

$z = \begin{bmatrix} 8.0 \\ 5.5 \\ 2.5 \end{bmatrix} + \begin{bmatrix} 0.1 \\ 2 \\ -0.5 \end{bmatrix} = \begin{bmatrix} 8.1 \\ 7.5 \\ 2.0 \end{bmatrix}$

$A = \text{ReLU}(z) = \begin{bmatrix} \max(0,\, 8.1) \\ \max(0,\, 7.5) \\ \max(0,\, 2.0) \end{bmatrix} = \begin{bmatrix} 8.1 \\ 7.5 \\ 2.0 \end{bmatrix}$

$Z^L = W^L A^{L-1} + b^L, \qquad A^L = g(Z^L)$

$\text{ReLU}(z) = \max(0,\, z)$

$\text{Leaky ReLU}(z) = \begin{cases} z & \text{if } z > 0 \\ \alpha z & \text{if } z \le 0 \end{cases}$

$\sigma(z) = \frac{1}{1 + e^{-z}}$

$\tanh(z) = \frac{e^z - e^{-z}}{e^z + e^{-z}}$

$Z^1 = W^1 x + b^1 = \begin{bmatrix} 2 & 7 \\ 3 & 4 \end{bmatrix} \begin{bmatrix} 0.5 \\ 1.0 \end{bmatrix} + \begin{bmatrix} 0.1 \\ 2 \end{bmatrix}$

$W^1 x = \begin{bmatrix} 2(0.5) + 7(1.0) \\ 3(0.5) + 4(1.0) \end{bmatrix} = \begin{bmatrix} 1.0 + 7.0 \\ 1.5 + 4.0 \end{bmatrix} = \begin{bmatrix} 8.0 \\ 5.5 \end{bmatrix}$

$Z^1 = \begin{bmatrix} 8.0 \\ 5.5 \end{bmatrix} + \begin{bmatrix} 0.1 \\ 2 \end{bmatrix} = \begin{bmatrix} 8.1 \\ 7.5 \end{bmatrix}$

$A^1 = \text{ReLU}(Z^1) = \begin{bmatrix} \max(0, 8.1) \\ \max(0, 7.5) \end{bmatrix} = \begin{bmatrix} 8.1 \\ 7.5 \end{bmatrix}$

$Z^2 = W^2 A^1 + b^2 = \begin{bmatrix} 1 & -1 \end{bmatrix} \begin{bmatrix} 8.1 \\ 7.5 \end{bmatrix} + \begin{bmatrix} 0.5 \end{bmatrix}$

$Z^2 = 1(8.1) + (-1)(7.5) + 0.5 = 8.1 - 7.5 + 0.5 = 1.1$

$A^2 = Z^2 = 1.1$

$Z^L = W^L A^{L-1} + b^L$

$A^L = g\!\left(Z^L\right)$

$\forall\, \varepsilon > 0,\; \exists\, (\mathbf{w}, \mathbf{b}) \text{ such that } \sup_{x \in D} |\,\text{NN}(x;\, \mathbf{w}, \mathbf{b}) - f(x)\,| < \varepsilon$

$Z^l = W^l \, A^{l-1} + b^l$

$A^l = G(Z^l)$

$\text{MSE} = \frac{1}{N} \sum_{i=1}^{N} (y_i - \hat{y}_i)^2$

$\mathcal{L}_{CE} = -\sum_{c=1}^{C} y_c \log \hat{y}_c$

$\theta_{\text{new}} = \theta_{\text{old}} - \alpha \cdot \frac{dL}{d\theta}$

$M \leftarrow M - \alpha \cdot \frac{\partial L}{\partial M}, \quad C \leftarrow C - \alpha \cdot \frac{\partial L}{\partial C}$

$\frac{\partial L}{\partial C} = \frac{\partial L}{\partial L} \cdot \frac{\partial L}{\partial C} = 1 \cdot D = 1 \cdot 5 = 5$

$\frac{\partial L}{\partial D} = \frac{\partial L}{\partial L} \cdot C = 1 \cdot 6 = 6$

$\frac{\partial D}{\partial A} = 1 \qquad \frac{\partial D}{\partial B} = 1$

$\frac{\partial L}{\partial A} = \frac{\partial L}{\partial D} \cdot \frac{\partial D}{\partial A} = 6 \cdot 1 = 6$

$\delta = \frac{\partial L}{\partial Z} = \frac{\partial L}{\partial A} \odot \frac{\partial A}{\partial Z} = \frac{\partial L}{\partial A} \odot g'(Z)$

$\frac{\partial L}{\partial W} = \delta \cdot x = 1 \times 2 = 2$

$\frac{\partial L}{\partial b} = \delta \cdot 1 = 1 \times 1 = 1$

$\frac{\partial L}{\partial A} = \begin{bmatrix} 2A_1 \\ 2A_2 \end{bmatrix} = \begin{bmatrix} 2 \times 7.5 \\ 2 \times 4 \end{bmatrix} = \begin{bmatrix} 15 \\ 8 \end{bmatrix}$

$\delta = \frac{\partial L}{\partial A} \odot g'(Z) = \begin{bmatrix} 15 \\ 8 \end{bmatrix} \odot \begin{bmatrix} 1 \\ 1 \end{bmatrix} = \begin{bmatrix} 15 \\ 8 \end{bmatrix}$

$\frac{\partial L}{\partial W} = \delta \cdot x^T = \begin{bmatrix} 15 \\ 8 \end{bmatrix} \begin{bmatrix} 2 & 1 \end{bmatrix} = \begin{bmatrix} 15 \times 2 & 15 \times 1 \\ 8 \times 2 & 8 \times 1 \end{bmatrix} = \begin{bmatrix} 30 & 15 \\ 16 & 8 \end{bmatrix}$

$\frac{\partial L}{\partial A^2} = 2 A^2 = 2 \times 14 = 28$

$\delta^2 = \frac{\partial L}{\partial A^2} \cdot g'(Z^2) = 28 \times 1 = 28$

$\frac{\partial L}{\partial A^1} = (W^2)^T \cdot \delta^2 = 2 \times 28 = 56$

$\delta^1 = \frac{\partial L}{\partial A^1} \cdot g'(Z^1) = 56 \times 1 = 56$

$\delta^L = (W^{L+1})^T \delta^{L+1} \odot g'(Z^L)$

$f(x_1, x_2) = w_1 x_1 + w_2 x_2 + b$

$z_1 = w_{11}\, x_1 + w_{21}\, x_2 + b_1, \qquad h_1 = \sigma(z_1)$

$z_2 = w_{12}\, x_1 + w_{22}\, x_2 + b_2, \qquad h_2 = \sigma(z_2)$

$z_3 = w_{31}\, h_1 + w_{32}\, h_2 + b_3$

$\hat{y} = \sigma(z_3) = \frac{1}{1 + e^{-(-0.038)}} = \frac{1}{1 + e^{0.038}} \approx \frac{1}{1 + 1.0388} \approx 0.491$

$L = -\bigl[y \cdot \log(\hat{y}) + (1-y) \cdot \log(1-\hat{y})\bigr]$

$\delta_3 = \hat{y} - y$

$\frac{\partial L}{\partial w_{31}} = \delta_3 \cdot h_1 = -0.509 \times 0.731 \approx -0.372$

$\frac{\partial L}{\partial w_{32}} = \delta_3 \cdot h_2 = -0.509 \times 0.5 \approx -0.255$

$w_{\text{new}} = w_{\text{old}} - \alpha \cdot \frac{\partial L}{\partial w}$

W_{ ext{new}} = W_{ ext{old}} - ext{lr} \cdot rac{\partial ext{Loss}}{\partial W}

$H_{i,j}^{(c_o)} = \sum_{c_i} \sum_{a,b} W_{c_i,\,c_o,\,a,\,b} \; X_{i+a,\;j+b}^{(c_i)}$

$\text{output size} = \text{input size} - \text{filter size} + 1$

$\text{Receptive field (Layer 1)} = \text{Filter size} = F \times F$

$\text{output size} = \text{input size} - K + 1$

$\text{output size} = \left\lfloor \frac{\text{input size} - K}{S} \right\rfloor + 1$

$\text{output size} = \left\lfloor \frac{W - K + 2P}{S} \right\rfloor + 1$

$\mu_B = \frac{1}{m} \sum_{i=1}^{m} x_i = \frac{2 + 4 + 3 + 3}{4} = \frac{12}{4} = 3$

$\sigma^2_B = \frac{1}{m} \sum_{i=1}^{m}(x_i - \mu_B)^2 = \frac{(2-3)^2+(4-3)^2+(3-3)^2+(3-3)^2}{4} = \frac{1+1+0+0}{4} = 0.5$

$\hat{x}_i = \frac{x_i - \mu_B}{\sqrt{\sigma^2_B + \varepsilon}}$

$y_i = \gamma \hat{x}_i + \beta$

$\mu_{\text{running}} \leftarrow (1 - \alpha)\,\mu_{\text{running}} + \alpha\,\mu_B$

$\sigma^2_{\text{running}} \leftarrow (1 - \alpha)\,\sigma^2_{\text{running}} + \alpha\,\sigma^2_B$

$\hat{x} = \frac{x - \mu_{\text{running}}}{\sqrt{\sigma^2_{\text{running}} + \varepsilon}}, \quad y = \gamma \hat{x} + \beta$

$y_i = \begin{cases} \dfrac{x_i}{1-p} & \text{with probability } 1-p \\ 0 & \text{with probability } p \end{cases}$

$y_i = x_i \quad \text{(inference mode, no dropout, no scaling)}$

$H_{\text{out}} = \frac{H_{\text{in}} - K + 2P}{S} + 1$

$\text{Parameters} = (K_H \times K_W \times C_{\text{in}} + 1) \times C_{\text{out}}$

$\mathcal{L} = \underbrace{L_{\text{target}}\!\bigl(y_t,\, f(\theta;\, \phi(x_t))\bigr)}_{\text{adaptation term}} + \underbrace{\lambda\, R(\phi)}_{\text{preservation term}}$

$x_{\text{norm}} = \frac{x_{\text{float}} - \mu}{\sigma}$

$\text{trainable params} = (1280 \times 3) + 3 = 3840 + 3 = 3843$

$\mathcal{L}_{\text{CE}} = -\log\left(\frac{e^{s_{y}}}{\sum_{c=1}^{3} e^{s_c}}\right)$

$\text{accuracy} = \frac{1}{N}\sum_{i=1}^{N} \mathbf{1}\!\left[\hat{y}_i = y_i\right]$