diff --git a/defaults/main.yml b/defaults/main.yml
index 4d525e9..3b7e20a 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -55,6 +55,7 @@ rally_pip_packages:
   - setuptools
   - pymysql
   - rally
+  - ansible==2.3.2.0
 
 # The inventory group where Rally will be installed.
 # This variable is used by the repo_build process to determine
diff --git a/tests/test-functional-rally.yml b/tests/test-functional-rally.yml
index 5e124c7..5b10626 100644
--- a/tests/test-functional-rally.yml
+++ b/tests/test-functional-rally.yml
@@ -26,5 +26,8 @@
       shell: |
         . {{ rally_bin }}/activate
         {{ rally_bin }}/rally task start /var/tmp/scenario.json
+      changed_when: true
+      tags:
+        - skip_ansible_lint
   vars_files:
     - common/test-vars.yml