update dropout

2024-11-19 17:08:52 +08:00 · 2024-11-19 17:08:52 +08:00 · 3cc902bead
parent 1fc98152c0
commit 3cc902bead
1 changed files with 1 additions and 1 deletions
--- a/model/model.py
+++ b/model/model.py
@ -430,9 +430,9 @@ class Transformer(PreTrainedModel):
        _bsz, seqlen = tokens.shape
        # language proj token
        h = self.tok_embeddings(tokens)
-        h = self.dropout(h)
        # vision proj token
        h = self.count_vision_proj(tokens=tokens, h=h, image_encoders=image_encoders, seqlen=seqlen)
+        h = self.dropout(h)

        pos_cis = self.pos_cis[current_idx:current_idx + seqlen]
        for idx, layer in enumerate(self.layers):