diff --git a/doc/source/contributor/periodic-work.rst b/doc/source/contributor/periodic-work.rst
index e96024dd6f..50b0e79518 100644
--- a/doc/source/contributor/periodic-work.rst
+++ b/doc/source/contributor/periodic-work.rst
@@ -52,10 +52,6 @@ The update script is used as follows:
    # the console code should only be updated when necessary for a security fix, or for the OSA master branch
    ./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/nova_consoles.yml -b master
 
-   # the testing repositories should not be updated for stable branches as the new tests
-   # or other changes introduced may not work for older branches
-   ./scripts/sources-branch-updater.sh -s playbooks/defaults/repo_packages/openstack_testing.yml -b master
-
    # commit the changes
    new_version=$(awk '/^openstack_release/ {print $2}' inventory/group_vars/all/all.yml)
    git add --all
diff --git a/playbooks/defaults/repo_packages/openstack_testing.yml b/playbooks/defaults/repo_packages/openstack_testing.yml
deleted file mode 100644
index d77e686b4c..0000000000
--- a/playbooks/defaults/repo_packages/openstack_testing.yml
+++ /dev/null
@@ -1,32 +0,0 @@
----
-# Copyright 2014, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-## NOTICE on items in this file:
-##   * If you use anything in the *._git_install_branch field that is not a TAG
-##     make sure to leave an in-line comment as to "why".
-
-## For the sake of anyone else editing this file:
-##   * If you add clients to this file please do so in alphabetical order.
-##   * Every entry should be name spaced with the name of the client followed by an "_"
-##   * All items with this file should be separated by `name_` note that the name of the
-##     package should be one long name with no additional `_` separating it.
-
-
-## Rally service
-rally_git_repo: https://git.openstack.org/openstack/rally
-rally_git_install_branch: 4c687b50d97841f131c9f375d708e3fb59e6915e # HEAD of "master" as of 31.03.2018
-rally_git_project_group: utility_all
-rally_git_install_fragments: "venvwithindex=True&ignorerequirements=True"
diff --git a/playbooks/os-tempest-install.yml b/playbooks/os-tempest-install.yml
index 820a6976b2..ea871dc73d 100644
--- a/playbooks/os-tempest-install.yml
+++ b/playbooks/os-tempest-install.yml
@@ -30,8 +30,6 @@
         rsyslog_client_log_dir: "{{ tempest_log_dir }}"
         rsyslog_client_config_name: "99-tempest-rsyslog-client.conf"
 
-  vars_files:
-    - defaults/repo_packages/openstack_testing.yml
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
     - tempest
diff --git a/releasenotes/notes/build-rally-with-constraints-60e12254103018c4.yaml b/releasenotes/notes/build-rally-with-constraints-60e12254103018c4.yaml
new file mode 100644
index 0000000000..3906b32a80
--- /dev/null
+++ b/releasenotes/notes/build-rally-with-constraints-60e12254103018c4.yaml
@@ -0,0 +1,11 @@
+---
+features:
+  - When ``venvwithindex=True`` and ``ignorerequirements=True`` are both specified
+    in ``rally_git_install_fragments`` (as was previously the default), this
+    results in rally being installed from PyPI without any constraints being
+    applied. This results in inconsistent builds from day to day, and can cause
+    build failures for stable implementations due to new library releases.
+    Going forward, we remove the ``rally_git_*`` overrides in
+    ``playbooks/defaults/repo_packages/openstack_testing.yml`` so that
+    the integrated build installs rally from PyPI, but with appropriate
+    constraints applied.