diff --git a/tests/roles/__init__.py b/tests/roles/__init__.py
new file mode 100644
index 0000000..603655d
--- /dev/null
+++ b/tests/roles/__init__.py
@@ -0,0 +1,3 @@
+# Stestr discovery in zuul-jobs requires a fully importable filesystem
+# hierarchy, so simulate that by including a __init__.py file in the
+# roles dir.
diff --git a/zuul_sphinx/zuul.py b/zuul_sphinx/zuul.py
index 114f0d3..7fb3800 100644
--- a/zuul_sphinx/zuul.py
+++ b/zuul_sphinx/zuul.py
@@ -180,6 +180,8 @@ class ZuulDirective(Directive):
         roles = env.domaindata['zuul']['role_paths']
         for d in role_dirs:
             for p in os.listdir(d):
+                if not os.path.isdir(os.path.join(d, p)):
+                    continue
                 role_readme = os.path.join(d, p, 'README.rst')
                 if os.path.exists(role_readme):
                     roles[p] = role_readme