From 51c813001b3db4ecde0a70ab7896f2203f745a8b Mon Sep 17 00:00:00 2001
From: Haoxiang Li <haoxiang.li@localhost>
Date: Wed, 4 Oct 2023 18:04:12 -0700
Subject: [PATCH] use pytorch built-in SiLU function to save GPU memory usage

---
 ldm/modules/diffusionmodules/model.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ldm/modules/diffusionmodules/model.py b/ldm/modules/diffusionmodules/model.py
index b089eeb..5173f89 100644
--- a/ldm/modules/diffusionmodules/model.py
+++ b/ldm/modules/diffusionmodules/model.py
@@ -5,6 +5,7 @@ import torch.nn as nn
 import numpy as np
 from einops import rearrange
 from typing import Optional, Any
+import torch.nn.functional as F
 
 from ldm.modules.attention import MemoryEfficientCrossAttention
 
@@ -40,7 +41,7 @@ def get_timestep_embedding(timesteps, embedding_dim):
 
 def nonlinearity(x):
     # swish
-    return x*torch.sigmoid(x)
+    return F.silu(x)
 
 
 def Normalize(in_channels, num_groups=32):