Refactor Serializers for different formats and request types

This restructures the entire API and allows it to return data in a format
that is easier to use by humans and tools.

TL;DR:
- List serializers (ex: GET /api/v1/playbooks) return a lightweight list
  of objects without detailed relationships.
- Detailed serializers (ex: GET /api/v1/playbooks/1) return every fields
  of an object with detailed relationships.
- Nested serializers are used by detailed serializers to represent complex
  structures (ex: playbook -> play -> tasks -> result <- host)
- Simple serializers are used by detailed serializers when representing
  a simple parent/children.
- Default serializers are used to manage requests for creating, updating
  and destroying objects.

Change-Id: I7f3badb48c6036a5708260541d2ea36c7f29e512
This commit is contained in:
David Moreau Simard 2019-02-22 18:17:44 -05:00
parent e4519339b6
commit 246f472370
5 changed files with 439 additions and 83 deletions

View File

@ -21,13 +21,10 @@ from ara.api import fields as ara_fields, models
class DurationSerializer(serializers.ModelSerializer):
"""
Serializer for duration-based fields
"""
class Meta:
abstract = True
# For objects that occur over a period of time
duration = serializers.SerializerMethodField()
@staticmethod
@ -37,35 +34,358 @@ class DurationSerializer(serializers.ModelSerializer):
return obj.ended - obj.started
class FileSerializer(serializers.ModelSerializer):
class ItemCountSerializer(serializers.ModelSerializer):
class Meta:
model = models.File
fields = "__all__"
abstract = True
# For counting relationships to other objects
items = serializers.SerializerMethodField()
@staticmethod
def get_items(obj):
types = ["plays", "tasks", "results", "hosts", "files", "records"]
items = {item: getattr(obj, item).count() for item in types if hasattr(obj, item)}
return items
class FileSha1Serializer(serializers.ModelSerializer):
class Meta:
abstract = True
# For retrieving the sha1 of a file's contents
sha1 = serializers.SerializerMethodField()
content = ara_fields.FileContentField()
@staticmethod
def get_sha1(obj):
return obj.content.sha1
def get_unique_together_validators(self):
"""
Files have a "unique together" constraint for file.path and playbook.id.
We want to have a "get_or_create" facility and in order to do that, we
must manage the validation during the creation, not before.
Overriding this method effectively disables this validator.
"""
return []
#######
# Simple serializers provide lightweight representations of objects without
# nested or large fields.
#######
class SimpleLabelSerializer(serializers.ModelSerializer):
class Meta:
model = models.Label
exclude = ("description", "created", "updated")
class SimplePlaybookSerializer(DurationSerializer):
class Meta:
model = models.Playbook
exclude = ("arguments", "labels", "ansible_version", "created", "updated")
class SimplePlaySerializer(DurationSerializer):
class Meta:
model = models.Play
exclude = ("uuid", "created", "updated")
class SimpleTaskSerializer(DurationSerializer):
class Meta:
model = models.Task
exclude = ("tags", "created", "updated")
class SimpleResultSerializer(DurationSerializer):
class Meta:
model = models.Result
exclude = ("content", "created", "updated")
class SimpleHostSerializer(serializers.ModelSerializer):
class Meta:
model = models.Host
exclude = ("facts", "created", "updated")
class SimpleFileSerializer(FileSha1Serializer):
class Meta:
model = models.File
exclude = ("content", "created", "updated")
class SimpleRecordSerializer(serializers.ModelSerializer):
class Meta:
model = models.Record
exclude = ("value", "created", "updated")
#######
# Nested serializers returns optimized data within the context of another object.
# For example: when retrieving a playbook, we'll already have the playbook id
# so it is not necessary to include it in nested objects.
#######
class NestedPlaybookFileSerializer(serializers.ModelSerializer):
class Meta:
model = models.File
exclude = ("content", "created", "updated", "playbook")
class NestedPlaybookHostSerializer(serializers.ModelSerializer):
class Meta:
model = models.Host
fields = ("id", "name", "alias")
class NestedPlaybookResultSerializer(DurationSerializer):
class Meta:
model = models.Result
exclude = ("content", "created", "updated", "playbook", "play", "task")
host = NestedPlaybookHostSerializer(read_only=True)
class NestedPlaybookTaskSerializer(DurationSerializer):
class Meta:
model = models.Task
exclude = ("playbook", "created", "updated")
tags = ara_fields.CompressedObjectField(read_only=True)
results = NestedPlaybookResultSerializer(read_only=True, many=True)
file = NestedPlaybookFileSerializer(read_only=True)
class NestedPlaybookRecordSerializer(serializers.ModelSerializer):
class Meta:
model = models.Record
exclude = ("playbook", "value", "created", "updated")
class NestedPlaybookPlaySerializer(DurationSerializer):
class Meta:
model = models.Play
exclude = ("playbook", "uuid", "created", "updated")
tasks = NestedPlaybookTaskSerializer(read_only=True, many=True)
class NestedPlayTaskSerializer(DurationSerializer):
class Meta:
model = models.Task
exclude = ("playbook", "play", "created", "updated")
tags = ara_fields.CompressedObjectField(read_only=True)
results = NestedPlaybookResultSerializer(read_only=True, many=True)
file = NestedPlaybookFileSerializer(read_only=True)
#######
# Detailed serializers returns every field of an object as well as a simple
# representation of relationships to other objects.
#######
class DetailedLabelSerializer(serializers.ModelSerializer):
class Meta:
model = models.Label
fields = "__all__"
description = ara_fields.CompressedTextField(read_only=True)
class DetailedPlaybookSerializer(DurationSerializer, ItemCountSerializer):
class Meta:
model = models.Playbook
fields = "__all__"
arguments = ara_fields.CompressedObjectField(default=ara_fields.EMPTY_DICT, read_only=True)
labels = SimpleLabelSerializer(many=True, read_only=True, default=[])
plays = NestedPlaybookPlaySerializer(many=True, read_only=True, default=[])
hosts = SimpleHostSerializer(many=True, read_only=True, default=[])
files = SimpleFileSerializer(many=True, read_only=True, default=[])
records = NestedPlaybookRecordSerializer(many=True, read_only=True, default=[])
class DetailedPlaySerializer(DurationSerializer, ItemCountSerializer):
class Meta:
model = models.Play
fields = "__all__"
playbook = SimplePlaybookSerializer(read_only=True)
tasks = NestedPlayTaskSerializer(many=True, read_only=True, default=[])
class DetailedTaskSerializer(DurationSerializer, ItemCountSerializer):
class Meta:
model = models.Task
fields = "__all__"
playbook = SimplePlaybookSerializer(read_only=True)
play = SimplePlaySerializer(read_only=True)
file = SimpleFileSerializer(read_only=True)
results = NestedPlaybookResultSerializer(many=True, read_only=True, default=[])
tags = ara_fields.CompressedObjectField(read_only=True)
class DetailedHostSerializer(serializers.ModelSerializer):
class Meta:
model = models.Host
fields = "__all__"
playbook = SimplePlaybookSerializer(read_only=True)
facts = ara_fields.CompressedObjectField(read_only=True)
class DetailedResultSerializer(serializers.ModelSerializer):
class Meta:
model = models.Result
fields = "__all__"
playbook = SimplePlaybookSerializer(read_only=True)
play = SimplePlaySerializer(read_only=True)
task = SimpleTaskSerializer(read_only=True)
host = SimpleHostSerializer(read_only=True)
content = ara_fields.CompressedObjectField(read_only=True)
class DetailedFileSerializer(FileSha1Serializer):
class Meta:
model = models.File
fields = "__all__"
playbook = SimplePlaybookSerializer(read_only=True)
content = ara_fields.FileContentField(read_only=True)
class DetailedRecordSerializer(serializers.ModelSerializer):
class Meta:
model = models.Record
fields = "__all__"
playbook = SimplePlaybookSerializer(read_only=True)
value = ara_fields.CompressedObjectField(read_only=True)
#######
# List serializers returns lightweight fields about objects.
# Relationships are represented by numerical IDs.
#######
class ListLabelSerializer(serializers.ModelSerializer):
class Meta:
model = models.Label
fields = "__all__"
description = ara_fields.CompressedTextField(
default=ara_fields.EMPTY_STRING, help_text="A text description of the label"
)
class ListPlaybookSerializer(DurationSerializer, ItemCountSerializer):
class Meta:
model = models.Playbook
exclude = ("arguments", "created", "updated")
labels = SimpleLabelSerializer(many=True, read_only=True, default=[])
class ListPlaySerializer(DurationSerializer, ItemCountSerializer):
class Meta:
model = models.Play
exclude = ("created", "updated")
playbook = serializers.PrimaryKeyRelatedField(read_only=True)
class ListTaskSerializer(DurationSerializer, ItemCountSerializer):
class Meta:
model = models.Task
exclude = ("created", "updated")
tags = ara_fields.CompressedObjectField(read_only=True)
play = serializers.PrimaryKeyRelatedField(read_only=True)
class ListHostSerializer(serializers.ModelSerializer):
class Meta:
model = models.Host
exclude = ("facts", "created", "updated")
playbook = serializers.PrimaryKeyRelatedField(read_only=True)
class ListResultSerializer(DurationSerializer):
class Meta:
model = models.Result
exclude = ("content", "created", "updated")
playbook = serializers.PrimaryKeyRelatedField(read_only=True)
play = serializers.PrimaryKeyRelatedField(read_only=True)
task = serializers.PrimaryKeyRelatedField(read_only=True)
host = serializers.PrimaryKeyRelatedField(read_only=True)
class ListFileSerializer(FileSha1Serializer):
class Meta:
model = models.File
exclude = ("content", "created", "updated")
playbook = serializers.PrimaryKeyRelatedField(read_only=True)
class ListRecordSerializer(serializers.ModelSerializer):
class Meta:
model = models.Record
exclude = ("value", "created", "updated")
playbook = serializers.PrimaryKeyRelatedField(read_only=True)
#######
# Default serializers represents objects as they are modelized in the database.
# They are used for creating/updating/destroying objects.
#######
class LabelSerializer(serializers.ModelSerializer):
class Meta:
model = models.Label
fields = "__all__"
description = ara_fields.CompressedTextField(
default=ara_fields.EMPTY_STRING, help_text="A text description of the label"
)
class PlaybookSerializer(DurationSerializer):
class Meta:
model = models.Playbook
fields = "__all__"
arguments = ara_fields.CompressedObjectField(default=ara_fields.EMPTY_DICT)
labels = LabelSerializer(many=True, default=[])
def create(self, validated_data):
file_, created = models.File.objects.get_or_create(
path=validated_data["path"],
content=validated_data["content"],
playbook=validated_data["playbook"],
defaults=validated_data,
)
return file_
# First create the playbook without the labels
labels = validated_data.pop("labels")
playbook = models.Playbook.objects.create(**validated_data)
# Now associate the labels to the playbook
for label in labels:
playbook.labels.add(models.Label.objects.create(**label))
return playbook
class PlaySerializer(DurationSerializer):
class Meta:
model = models.Play
fields = "__all__"
class TaskSerializer(DurationSerializer):
class Meta:
model = models.Task
fields = "__all__"
tags = ara_fields.CompressedObjectField(default=ara_fields.EMPTY_LIST, help_text="A list containing Ansible tags")
class HostSerializer(serializers.ModelSerializer):
@ -99,28 +419,30 @@ class ResultSerializer(DurationSerializer):
content = ara_fields.CompressedObjectField(default=ara_fields.EMPTY_DICT)
class LabelSerializer(serializers.ModelSerializer):
class FileSerializer(FileSha1Serializer):
class Meta:
model = models.Label
model = models.File
fields = "__all__"
description = ara_fields.CompressedTextField(
default=ara_fields.EMPTY_STRING, help_text="A text description of the label"
)
content = ara_fields.FileContentField()
def get_unique_together_validators(self):
"""
Files have a "unique together" constraint for file.path and playbook.id.
We want to have a "get_or_create" facility and in order to do that, we
must manage the validation during the creation, not before.
Overriding this method effectively disables this validator.
"""
return []
class TaskSerializer(DurationSerializer):
class Meta:
model = models.Task
fields = "__all__"
tags = ara_fields.CompressedObjectField(default=ara_fields.EMPTY_LIST, help_text="A list containing Ansible tags")
class SimpleTaskSerializer(serializers.ModelSerializer):
class Meta:
model = models.Task
fields = ("id", "name")
def create(self, validated_data):
file_, created = models.File.objects.get_or_create(
path=validated_data["path"],
content=validated_data["content"],
playbook=validated_data["playbook"],
defaults=validated_data,
)
return file_
class RecordSerializer(serializers.ModelSerializer):
@ -131,32 +453,3 @@ class RecordSerializer(serializers.ModelSerializer):
value = ara_fields.CompressedObjectField(
default=ara_fields.EMPTY_STRING, help_text="A string, list, dict, json or other formatted data"
)
class PlaySerializer(DurationSerializer):
class Meta:
model = models.Play
fields = "__all__"
hosts = HostSerializer(read_only=True, many=True)
results = ResultSerializer(read_only=True, many=True)
class SimplePlaySerializer(serializers.ModelSerializer):
class Meta:
model = models.Play
fields = "__all__"
class PlaybookSerializer(DurationSerializer):
class Meta:
model = models.Playbook
fields = "__all__"
arguments = ara_fields.CompressedObjectField(default=ara_fields.EMPTY_DICT)
files = FileSerializer(many=True, read_only=True, default=[])
hosts = HostSerializer(many=True, read_only=True, default=[])
labels = LabelSerializer(many=True, read_only=True, default=[])
tasks = SimpleTaskSerializer(many=True, read_only=True, default=[])
plays = SimplePlaySerializer(many=True, read_only=True, default=[])
records = RecordSerializer(many=True, read_only=True, default=[])

View File

@ -15,11 +15,15 @@
# You should have received a copy of the GNU General Public License
# along with ARA. If not, see <http://www.gnu.org/licenses/>.
import logging
import factory
from ara.api import models
from ara.api.tests import utils
logging.getLogger("factory").setLevel(logging.INFO)
# constants for things like compressed byte strings or objects
FILE_CONTENTS = "---\n# Example file"
HOST_FACTS = {"ansible_fqdn": "hostname", "ansible_distribution": "CentOS"}

View File

@ -65,11 +65,6 @@ class PlaybookTestCase(APITestCase):
self.assertEqual(1, request.data["count"])
playbook = request.data["results"][0]
self.assertEqual(playbook["ansible_version"], expected_playbook.ansible_version)
self.assertEqual(len(playbook["files"]), 0)
self.assertEqual(len(playbook["hosts"]), 0)
self.assertEqual(len(playbook["tasks"]), 0)
self.assertEqual(len(playbook["records"]), 0)
self.assertEqual(len(playbook["plays"]), 0)
def test_delete_playbook(self):
playbook = factories.PlaybookFactory()

View File

@ -35,35 +35,74 @@ class InfoView(viewsets.ViewSet):
class LabelViewSet(viewsets.ModelViewSet):
queryset = models.Label.objects.all()
serializer_class = serializers.LabelSerializer
def get_serializer_class(self):
if self.action == "list":
return serializers.ListLabelSerializer
elif self.action == "retrieve":
return serializers.DetailedLabelSerializer
else:
# create/update/destroy
return serializers.LabelSerializer
class PlaybookViewSet(viewsets.ModelViewSet):
queryset = models.Playbook.objects.all()
serializer_class = serializers.PlaybookSerializer
filter_fields = ("name", "status")
def get_serializer_class(self):
if self.action == "list":
return serializers.ListPlaybookSerializer
elif self.action == "retrieve":
return serializers.DetailedPlaybookSerializer
else:
# create/update/destroy
return serializers.PlaybookSerializer
class PlayViewSet(viewsets.ModelViewSet):
queryset = models.Play.objects.all()
serializer_class = serializers.PlaySerializer
filter_fields = ("playbook", "uuid")
def get_serializer_class(self):
if self.action == "list":
return serializers.ListPlaySerializer
elif self.action == "retrieve":
return serializers.DetailedPlaySerializer
else:
# create/update/destroy
return serializers.PlaySerializer
class TaskViewSet(viewsets.ModelViewSet):
queryset = models.Task.objects.all()
serializer_class = serializers.TaskSerializer
filter_fields = ("playbook",)
def get_serializer_class(self):
if self.action == "list":
return serializers.ListTaskSerializer
elif self.action == "retrieve":
return serializers.DetailedTaskSerializer
else:
# create/update/destroy
return serializers.TaskSerializer
class HostViewSet(viewsets.ModelViewSet):
queryset = models.Host.objects.all()
serializer_class = serializers.HostSerializer
filter_fields = ("playbook",)
def get_serializer_class(self):
if self.action == "list":
return serializers.ListHostSerializer
elif self.action == "retrieve":
return serializers.DetailedHostSerializer
else:
# create/update/destroy
return serializers.HostSerializer
class ResultViewSet(viewsets.ModelViewSet):
serializer_class = serializers.ResultSerializer
filter_fields = ("playbook",)
def get_queryset(self):
@ -72,14 +111,39 @@ class ResultViewSet(viewsets.ModelViewSet):
return models.Result.objects.filter(status__in=statuses)
return models.Result.objects.all()
def get_serializer_class(self):
if self.action == "list":
return serializers.ListResultSerializer
elif self.action == "retrieve":
return serializers.DetailedResultSerializer
else:
# create/update/destroy
return serializers.ResultSerializer
class FileViewSet(viewsets.ModelViewSet):
queryset = models.File.objects.all()
serializer_class = serializers.FileSerializer
filter_fields = ("playbook", "path")
def get_serializer_class(self):
if self.action == "list":
return serializers.ListFileSerializer
elif self.action == "retrieve":
return serializers.DetailedFileSerializer
else:
# create/update/destroy
return serializers.FileSerializer
class RecordViewSet(viewsets.ModelViewSet):
queryset = models.Record.objects.all()
serializer_class = serializers.RecordSerializer
filter_fields = ("playbook", "key")
def get_serializer_class(self):
if self.action == "list":
return serializers.ListRecordSerializer
elif self.action == "retrieve":
return serializers.DetailedRecordSerializer
else:
# create/update/destroy
return serializers.RecordSerializer

View File

@ -158,7 +158,7 @@ class ActionModule(ActionBase):
changed = True
else:
# Otherwise update it if the data is different (idempotency)
old = record["results"][0]
old = self.client.get("/api/v1/records/%s" % record["results"][0]["id"])
if old["value"] != value or old["type"] != type:
record = self.client.patch("/api/v1/records/%s" % old["id"], key=key, value=value, type=type)
changed = True