diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 00000000..68fa4741
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,43 @@
+name: Python tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    name: Run tests on ${{ matrix.os }} with Python ${{ matrix.python }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macOS-latest]
+        python: ['3.8']
+        torch: ['1.5.0']
+        torchvision: ['0.6.0']
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install testing dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest pytest-timeout
+    - name: Install torch on mac
+      if: startsWith(matrix.os, 'macOS')
+      run: pip install torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}
+    - name: Install torch on ubuntu
+      if: startsWith(matrix.os, 'ubuntu')
+      run: pip install torch==${{ matrix.torch }}+cpu torchvision==${{ matrix.torchvision }}+cpu -f https://download.pytorch.org/whl/torch_stable.html
+    - name: Install requirements
+      run: |
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install scipy
+        pip install git+https://github.com/mapillary/inplace_abn.git@v1.0.11
+    - name: Run tests
+      run: |
+        pytest -vv --durations=0 ./tests
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/test_inference.py b/tests/test_inference.py
new file mode 100644
index 00000000..75b8d445
--- /dev/null
+++ b/tests/test_inference.py
@@ -0,0 +1,19 @@
+import pytest
+import torch
+
+from timm import list_models, create_model
+
+
+@pytest.mark.timeout(60)
+@pytest.mark.parametrize('model_name', list_models())
+@pytest.mark.parametrize('batch_size', [1])
+def test_model_forward(model_name, batch_size):
+  """Run a single forward pass with each model"""
+  model = create_model(model_name, pretrained=False)
+  model.eval()
+
+  inputs = torch.randn((batch_size, *model.default_cfg['input_size']))
+  outputs = model(inputs)
+
+  assert outputs.shape[0] == batch_size
+  assert not torch.isnan(outputs).any(), 'Output included NaNs'