From b50a4acc1fb931cc384e606d60ca36bb477746c7 Mon Sep 17 00:00:00 2001 From: Martin Paulo Date: Thu, 26 Nov 2015 13:57:54 +1100 Subject: [PATCH] Added the jclouds durability code sample Added the durability code sample for jclouds and modified the durability chapter to reference it. Change-Id: Id177f18d4bc36642922d7196cde46adf71f20de6 Partial-Bug: #1449330 --- firstapp/samples/jclouds/Durability.java | 280 +++++++++++++++++++++++ firstapp/source/durability.rst | 145 +++++++++++- 2 files changed, 415 insertions(+), 10 deletions(-) create mode 100644 firstapp/samples/jclouds/Durability.java diff --git a/firstapp/samples/jclouds/Durability.java b/firstapp/samples/jclouds/Durability.java new file mode 100644 index 000000000..9e1aefca4 --- /dev/null +++ b/firstapp/samples/jclouds/Durability.java @@ -0,0 +1,280 @@ +import com.google.common.collect.ImmutableMap; +import com.google.common.io.ByteSource; +import com.google.common.io.Closeables; +import com.google.common.io.Files; +import com.google.gson.Gson; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.domain.Location; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; +import org.jclouds.openstack.swift.v1.SwiftApi; +import org.jclouds.openstack.swift.v1.blobstore.RegionScopedBlobStoreContext; +import org.jclouds.openstack.swift.v1.domain.Container; +import org.jclouds.openstack.swift.v1.domain.SwiftObject; +import org.jclouds.openstack.swift.v1.features.ContainerApi; +import org.jclouds.openstack.swift.v1.features.ObjectApi; +import org.jclouds.openstack.swift.v1.options.CreateContainerOptions; +import org.jclouds.openstack.swift.v1.options.PutOptions; + +import java.io.*; +import java.net.URL; +import java.net.URLConnection; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.List; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static java.lang.System.out; +import static org.jclouds.io.Payloads.newFilePayload; +import static org.jclouds.io.Payloads.newInputStreamPayload; + +public class Durability implements Closeable { + + // step-1 + private final SwiftApi swiftApi; + private final String region; + private static final String PROVIDER = "openstack-swift"; + + private static final String OS_AUTH_URL = "http://controller:5000/v2.0/"; + // format for identity is projectName:userName + private static final String IDENTITY = "your_project_name:your_auth_username"; + private static final String PASSWORD = "your_auth_password"; + + + public Durability() { + swiftApi = ContextBuilder.newBuilder(PROVIDER) + .endpoint(OS_AUTH_URL) + .credentials(IDENTITY, PASSWORD) + .buildApi(SwiftApi.class); + region = swiftApi.getConfiguredRegions().iterator().next(); + out.println("Running in region: " + region); + } + + // step-2 + private void createContainer(String containerName) { + ContainerApi containerApi = swiftApi.getContainerApi(region); + if (containerApi.create(containerName)) { + out.println("Created container: " + containerName); + } else { + out.println("Container all ready exists: " + containerName); + } + } + + // step-3 + private void listContainers() { + out.println("Containers:"); + ContainerApi containerApi = swiftApi.getContainerApi(region); + containerApi.list().forEach(container -> out.println(" " + container)); + } + + // step-4 + private String uploadObject(String containerName, String objectName, String filePath) { + Payload payload = newFilePayload(new File(filePath)); + ObjectApi objectApi = swiftApi.getObjectApi(region, containerName); + String eTag = objectApi.put(objectName, payload); + out.println(String.format("Uploaded %s as \"%s\" eTag = %s", filePath, objectName, eTag)); + return eTag; + } + + // step-5 + private void listObjectsInContainer(String containerName) { + out.println("Objects in " + containerName + ":"); + ObjectApi objectApi = swiftApi.getObjectApi(region, containerName); + objectApi.list().forEach(object -> out.println(" " + object)); + } + + // step-6 + private SwiftObject getObjectFromContainer(String containerName, String objectName) { + ObjectApi objectApi = swiftApi.getObjectApi(region, containerName); + SwiftObject object = objectApi.get(objectName); + out.println("Fetched: " + object.getName()); + return object; + } + + // step-7 + private void calculateMd5ForFile(String filePath) { + try (FileInputStream fis = new FileInputStream(new File(filePath))) { + MessageDigest md5Digest = MessageDigest.getInstance("MD5"); + + byte[] byteArray = new byte[1024]; + int bytesCount; + while ((bytesCount = fis.read(byteArray)) != -1) { + md5Digest.update(byteArray, 0, bytesCount); + } + byte[] digest = md5Digest.digest(); + + // Convert decimal number to hex string + StringBuilder sb = new StringBuilder(); + for (byte aByte : digest) { + sb.append(Integer.toString((aByte & 0xff) + 0x100, 16).substring(1)); + } + + out.println("MD5 for file " + filePath + ": " + sb.toString()); + } catch (IOException | NoSuchAlgorithmException e) { + out.println("Could not calculate md5: " + e.getMessage()); + } + } + + // step-8 + private void deleteObject(String containerName, String objectName) { + ObjectApi objectApi = swiftApi.getObjectApi(region, containerName); + objectApi.delete(objectName); + out.println("Deleted: " + objectName); + } + + // step-10 + private Container getContainer(String containerName) { + ContainerApi containerApi = swiftApi.getContainerApi(region); + // ensure container exists + containerApi.create(containerName); + return containerApi.get(containerName); + } + + // step-11 + static class Fractal { + // only included elements we want to work with + String uuid; + } + + static class Fractals { + // only included elements we want to work with + List objects; + } + + private void backupFractals(String containerName, String fractalsIp) { + // just need to make sure that there is container + getContainer(containerName); + try { + String response = ""; + String endpoint = "http://" + fractalsIp + "/v1/fractal"; + URLConnection connection = new URL(endpoint).openConnection(); + connection.setRequestProperty("'results_per_page", "-1"); + connection.getInputStream(); + try (BufferedReader in = new BufferedReader(new InputStreamReader( + connection.getInputStream()))) { + String inputLine; + while ((inputLine = in.readLine()) != null) { + response = response + inputLine; + } + } + + Gson gson = new Gson(); + Fractals fractals = gson.fromJson(response, Fractals.class); + ObjectApi objectApi = swiftApi.getObjectApi(region, containerName); + fractals.objects.forEach(fractal -> { + try { + String fractalEndpoint = "http://" + fractalsIp + "/fractal/" + fractal.uuid; + URLConnection conn = new URL(fractalEndpoint).openConnection(); + try (InputStream inputStream = conn.getInputStream()) { + Payload payload = newInputStreamPayload(inputStream); + String eTag = objectApi.put(fractal.uuid, payload); + out.println(String.format("Backed up %s eTag = %s", fractal.uuid, eTag)); + } + } catch (IOException e) { + out.println("Could not backup " + fractal.uuid + "! Cause: " + e.getMessage()); + } + }); + out.println("Backed up:"); + objectApi.list().forEach(object -> out.println(" " + object)); + } catch (IOException e) { + out.println("Could not backup fractals! Cause: " + e.getMessage()); + } + } + + // step-12 + private boolean deleteContainer(String containerName) { + ObjectApi objectApi = swiftApi.getObjectApi(region, containerName); + objectApi.list().forEach(object -> objectApi.delete(object.getName())); + ContainerApi containerApi = swiftApi.getContainerApi(region); + return containerApi.deleteIfEmpty(containerName); + } + + // step-13 + private void createWithMetadata(String containerName, String objectName, String filePath) { + + ContainerApi containerApi = swiftApi.getContainerApi(region); + CreateContainerOptions options = CreateContainerOptions.Builder + .metadata(ImmutableMap.of("photos", "of fractals")); + + if (containerApi.create(containerName, options)) { + out.println("Uploading: " + objectName); + + ObjectApi objectApi = swiftApi.getObjectApi(region, containerName); + Payload payload = newFilePayload(new File(filePath)); + PutOptions putOptions = PutOptions.Builder + .metadata(ImmutableMap.of( + "description", "a funny goat", + "created", "2015-06-02")); + String eTag = objectApi.put(objectName, payload, putOptions); + out.println( + String.format("Uploaded %s as \"%s\" eTag = %s", filePath, objectName, eTag)); + } else { + out.println("Could not upload " + objectName); + } + } + + // step-14 + private void uploadLargeFile(String containerName, String pathNameOfLargeFile) { + // Only works with jclouds V2 (in beta at the time of writing). See: + // https://issues.apache.org/jira/browse/JCLOUDS-894 + try { + RegionScopedBlobStoreContext context = ContextBuilder.newBuilder(PROVIDER) + .credentials(IDENTITY, PASSWORD) + .endpoint(OS_AUTH_URL) + .buildView(RegionScopedBlobStoreContext.class); + String region = context.getConfiguredRegions().iterator().next(); + out.println("Running in region: " + region); + BlobStore blobStore = context.getBlobStore(region); + // create the container if it doesn't exist... + Location location = getOnlyElement(blobStore.listAssignableLocations()); + blobStore.createContainerInLocation(location, containerName); + File largeFile = new File(pathNameOfLargeFile); + ByteSource source = Files.asByteSource(largeFile); + Payload payload = Payloads.newByteSourcePayload(source); + payload.getContentMetadata().setContentLength(largeFile.length()); + out.println("Uploading file. This may take some time!"); + Blob blob = blobStore.blobBuilder(largeFile.getName()) + .payload(payload) + .build(); + org.jclouds.blobstore.options.PutOptions putOptions = + new org.jclouds.blobstore.options.PutOptions(); + + String eTag = blobStore.putBlob(containerName, blob, putOptions.multipart()); + out.println(String.format("Uploaded %s eTag=%s", largeFile.getName(), eTag)); + } catch (UnsupportedOperationException e) { + out.println("Sorry: large file uploads only work in jclouds V2..."); + } + } + + // step-15 + @Override + public void close() throws IOException { + Closeables.close(swiftApi, true); + } + + public static void main(String[] args) throws IOException { + try (Durability tester = new Durability()) { + String containerName = "fractals"; + String objectName = "an amazing goat"; + String goatImageFilePath = "goat.jpg"; + String fractalsIp = "IP_API_1"; + String pathNameOfLargeFile = "big.img"; + + tester.createContainer(containerName); + tester.listContainers(); + tester.uploadObject(containerName, objectName, goatImageFilePath); + tester.listObjectsInContainer(containerName); + tester.getObjectFromContainer(containerName, objectName); + tester.calculateMd5ForFile(goatImageFilePath); + tester.deleteObject(containerName, objectName); + tester.getContainer(containerName); + tester.backupFractals(containerName, fractalsIp); + tester.deleteContainer(containerName); + tester.createWithMetadata(containerName, objectName, goatImageFilePath); + tester.listContainers(); + tester.uploadLargeFile("largeObject", pathNameOfLargeFile); + } + } +} diff --git a/firstapp/source/durability.rst b/firstapp/source/durability.rst index 41e0c7967..213ae4a7f 100644 --- a/firstapp/source/durability.rst +++ b/firstapp/source/durability.rst @@ -75,10 +75,12 @@ First, learn how to connect to the Object Storage endpoint: :start-after: step-1 :end-before: step-2 - .. only:: jclouds - .. warning:: This section has not yet been completed for the jclouds SDK. + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-1 + :end-before: step-2 .. only:: libcloud @@ -131,6 +133,13 @@ Call yours :code:`fractals`: TBC +.. only:: jclouds + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-2 + :end-before: step-3 + .. only:: libcloud .. literalinclude:: ../samples/libcloud/durability.py @@ -158,6 +167,13 @@ all containers in your account: TBC +.. only:: jclouds + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-3 + :end-before: step-4 + .. only:: libcloud .. literalinclude:: ../samples/libcloud/durability.py @@ -180,6 +196,13 @@ on line, name it :code:`goat.jpg`, and upload it to your :start-after: step-4 :end-before: step-5 +.. only:: jclouds + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-4 + :end-before: step-5 + .. only:: libcloud .. literalinclude:: ../samples/libcloud/durability.py @@ -217,6 +240,39 @@ the same: 7513986d3aeb22659079d1bf3dc2468b +.. only:: jclouds + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-5 + :end-before: step-6 + + :: + + Objects in fractals: + SwiftObject{name=an amazing goat, + uri=https://swift.some.org:8888/v1/AUTH_8997868/fractals/an%20amazing%20goat, + etag=439884df9c1c15c59d2cf43008180048, + lastModified=Wed Nov 25 15:09:34 AEDT 2015, metadata={}} + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-6 + :end-before: step-7 + + :: + + Fetched: an amazing goat + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-7 + :end-before: step-8 + + :: + + MD5 for file goat.jpg: 439884df9c1c15c59d2cf43008180048 + .. only:: libcloud @@ -254,6 +310,13 @@ Finally, clean up by deleting the test object: :start-after: step-8 :end-before: step-9 +.. only:: jclouds + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-8 + :end-before: step-10 + .. only:: libcloud .. literalinclude:: ../samples/libcloud/durability.py @@ -278,7 +341,7 @@ Back up the Fractals from the database on the Object Storage Back up the Fractals app images, which are currently stored inside the database, on Object Storage. -Place the images in the :code:`fractals`' container: +Place the images in the :code:`fractals` container: .. only:: fog @@ -286,6 +349,13 @@ Place the images in the :code:`fractals`' container: :start-after: step-10 :end-before: step-11 +.. only:: jclouds + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-10 + :end-before: step-11 + .. only:: libcloud .. literalinclude:: ../samples/libcloud/durability.py @@ -293,7 +363,7 @@ Place the images in the :code:`fractals`' container: :end-before: step-11 Next, back up all existing fractals from the database to the swift container. -A simple `for` loop takes care of that: +A simple loop takes care of that: .. note:: Replace :code:`IP_API_1` with the IP address of the API instance. @@ -303,6 +373,13 @@ A simple `for` loop takes care of that: :start-after: step-11 :end-before: step-12 +.. only:: jclouds + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-11 + :end-before: step-12 + .. only:: libcloud .. literalinclude:: ../samples/libcloud/durability.py @@ -336,8 +413,8 @@ Extra features Delete containers ~~~~~~~~~~~~~~~~~ -To delete a container, make sure that you have removed all objects from the -container before running this script. Otherwise, the script fails: +To delete a container, you must first remove all objects from the container. +Otherwise, the delete operation fails: .. only:: fog @@ -345,6 +422,13 @@ container before running this script. Otherwise, the script fails: :start-after: step-12 :end-before: step-13 +.. only:: jclouds + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-12 + :end-before: step-13 + .. only:: libcloud .. literalinclude:: ../samples/libcloud/durability.py @@ -358,19 +442,33 @@ Add metadata to objects You can complete advanced tasks such as uploading an object with metadata, as shown in following example. For more information, see the documentation for -your SDK. This option also uses a bit stream to upload the file, iterating bit -by bit over the file and passing those bits to Object Storage as they come. -Compared to loading the entire file in memory and then sending it, this method -is more efficient, especially for larger files. +your SDK. .. only:: fog + This option also uses a bit stream to upload the file, iterating bit + by bit over the file and passing those bits to Object Storage as they come. + Compared to loading the entire file in memory and then sending it, this method + is more efficient, especially for larger files. + .. literalinclude:: ../samples/fog/durability.rb :start-after: step-13 :end-before: step-14 +.. only:: jclouds + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-13 + :end-before: step-14 + .. only:: libcloud + This option also uses a bit stream to upload the file, iterating bit + by bit over the file and passing those bits to Object Storage as they come. + Compared to loading the entire file in memory and then sending it, this method + is more efficient, especially for larger files. + .. literalinclude:: ../samples/libcloud/durability.py :start-after: step-13 :end-before: step-14 @@ -389,6 +487,20 @@ For efficiency, most Object Storage installations treat large objects, :start-after: step-14 :end-before: step-15 +.. only:: jclouds + + If you work with large objects, use the :code:`RegionScopedBlobStoreContext` + class family instead of the ones used so far. + + .. note:: Large file uploads that use the :code:`openstack-swift` provider + are supported in only jclouds V2, currently in beta. Also, the + default chunk size is 64 Mb. Consider changing this as homework. + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java + :start-after: step-14 + :end-before: step-15 + .. only:: libcloud If you work with large objects, use the :code:`ex_multipart_upload_object` @@ -401,6 +513,19 @@ For efficiency, most Object Storage installations treat large objects, :start-after: step-14 :end-before: step-15 +.. only:: jclouds + + Complete code sample + ~~~~~~~~~~~~~~~~~~~~ + + This file contains all the code from this tutorial section. This + class lets you view and run the code. + + Before you run this class, confirm that you have configured it for + your cloud and the instance running the Fractals application. + + .. literalinclude:: ../samples/jclouds/Durability.java + :language: java Next steps ----------