diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 74a8173b519..47c902c117c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -75,7 +75,7 @@ jobs:
   checks:
     name: ${{ matrix.task.name }}
     runs-on: ${{ matrix.task.runs_on }}
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
diff --git a/tests/modules/transformer/t5_test.py b/tests/modules/transformer/t5_test.py
index 113649910d7..6aa2ab40038 100644
--- a/tests/modules/transformer/t5_test.py
+++ b/tests/modules/transformer/t5_test.py
@@ -3,16 +3,15 @@
 
 from allennlp.modules.transformer.t5 import T5
 from allennlp.nn.parallel import FairScaleFsdpAccelerator
-from allennlp.common.testing import requires_gpu, run_distributed_test, requires_multi_gpu
+from allennlp.common.testing import run_distributed_test, requires_multi_gpu
 
 
-# Mark this as GPU so it runs on a self-hosted runner, which will be a lot faster.
-@requires_gpu
+@pytest.mark.skip("takes too long in CI")
 @pytest.mark.parametrize(
     "pretrained_model_name",
     [
         "t5-base",
-        #  "t5-large",  # Takes too long in CI
+        #  "t5-large",  # Takes WAY too long in CI
     ],
 )
 def test_create_t5_from_pretrained(pretrained_model_name: str):