diff --git a/.travis.yml b/.travis.yml
index 299c239c..8c84c36c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,6 +32,7 @@ before_script:
   - flake8
   - ./bin/fetch-configlet
   - ./bin/configlet lint .
+  - ./bin/check-readmes.sh
 
 script:
   - ./test/check-exercises.py
diff --git a/bin/check-readmes.sh b/bin/check-readmes.sh
new file mode 100755
index 00000000..d23cefc1
--- /dev/null
+++ b/bin/check-readmes.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+get_timestamp()
+{
+    path="$1"
+    git log -n1 --pretty=format:%ct -- "$path"
+}
+
+ret=0
+for exercise in $(ls -d exercises/*/); do
+    meta_dir="${exercise}.meta"
+    if [ -d "$meta_dir" ]; then
+        meta_timestamp="$(get_timestamp "$meta_dir")"
+        readme_timestamp="$(get_timestamp "${exercise}README.md")"
+        if [ "$meta_timestamp" -gt "$readme_timestamp" ]; then
+            ret=1
+            echo "$(basename "$exercise"): .meta/ contents newer than README. Please regenerate it with configlet."
+        fi
+    fi
+done
+
+exit $ret