openstack/api-site
Code Issues Proposed changes

Added the jclouds durability code sample

Browse Source 
Added the durability code sample for jclouds and modified the
durability chapter to reference it.

Change-Id: Id177f18d4bc36642922d7196cde46adf71f20de6
Partial-Bug: #1449330
This commit is contained in:
Martin Paulo 2015-11-26 13:57:54 +11:00 committed by Diane Fleming
parent 8c93cdaf67
commit b50a4acc1f
2 changed files with 415 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

280
firstapp/samples/jclouds/Durability.java Normal file
View File

@ -0,0 +1,280 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.io.ByteSource;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
import com.google.gson.Gson;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.domain.Location;
import org.jclouds.io.Payload;
import org.jclouds.io.Payloads;
import org.jclouds.openstack.swift.v1.SwiftApi;
import org.jclouds.openstack.swift.v1.blobstore.RegionScopedBlobStoreContext;
import org.jclouds.openstack.swift.v1.domain.Container;
import org.jclouds.openstack.swift.v1.domain.SwiftObject;
import org.jclouds.openstack.swift.v1.features.ContainerApi;
import org.jclouds.openstack.swift.v1.features.ObjectApi;
import org.jclouds.openstack.swift.v1.options.CreateContainerOptions;
import org.jclouds.openstack.swift.v1.options.PutOptions;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import static com.google.common.collect.Iterables.getOnlyElement;
import static java.lang.System.out;
import static org.jclouds.io.Payloads.newFilePayload;
import static org.jclouds.io.Payloads.newInputStreamPayload;
public class Durability implements Closeable {
// step-1
private final SwiftApi swiftApi;
private final String region;
private static final String PROVIDER = "openstack-swift";
private static final String OS_AUTH_URL = "http://controller:5000/v2.0/";
// format for identity is projectName:userName
private static final String IDENTITY = "your_project_name:your_auth_username";
private static final String PASSWORD = "your_auth_password";
public Durability() {
swiftApi = ContextBuilder.newBuilder(PROVIDER)
.endpoint(OS_AUTH_URL)
.credentials(IDENTITY, PASSWORD)
.buildApi(SwiftApi.class);
region = swiftApi.getConfiguredRegions().iterator().next();
out.println("Running in region: " + region);
}
// step-2
private void createContainer(String containerName) {
ContainerApi containerApi = swiftApi.getContainerApi(region);
if (containerApi.create(containerName)) {
out.println("Created container: " + containerName);
} else {
out.println("Container all ready exists: " + containerName);
}
}
// step-3
private void listContainers() {
out.println("Containers:");
ContainerApi containerApi = swiftApi.getContainerApi(region);
containerApi.list().forEach(container -> out.println(" " + container));
}
// step-4
private String uploadObject(String containerName, String objectName, String filePath) {
Payload payload = newFilePayload(new File(filePath));
ObjectApi objectApi = swiftApi.getObjectApi(region, containerName);
String eTag = objectApi.put(objectName, payload);
out.println(String.format("Uploaded %s as \"%s\" eTag = %s", filePath, objectName, eTag));
return eTag;
}
// step-5
private void listObjectsInContainer(String containerName) {
out.println("Objects in " + containerName + ":");
ObjectApi objectApi = swiftApi.getObjectApi(region, containerName);
objectApi.list().forEach(object -> out.println(" " + object));
}
// step-6
private SwiftObject getObjectFromContainer(String containerName, String objectName) {
ObjectApi objectApi = swiftApi.getObjectApi(region, containerName);
SwiftObject object = objectApi.get(objectName);
out.println("Fetched: " + object.getName());
return object;
}
// step-7
private void calculateMd5ForFile(String filePath) {
try (FileInputStream fis = new FileInputStream(new File(filePath))) {
MessageDigest md5Digest = MessageDigest.getInstance("MD5");
byte[] byteArray = new byte[1024];
int bytesCount;
while ((bytesCount = fis.read(byteArray)) != -1) {
md5Digest.update(byteArray, 0, bytesCount);
}
byte[] digest = md5Digest.digest();
// Convert decimal number to hex string
StringBuilder sb = new StringBuilder();
for (byte aByte : digest) {
sb.append(Integer.toString((aByte & 0xff) + 0x100, 16).substring(1));
}
out.println("MD5 for file " + filePath + ": " + sb.toString());
} catch (IOException | NoSuchAlgorithmException e) {
out.println("Could not calculate md5: " + e.getMessage());
}
}
// step-8
private void deleteObject(String containerName, String objectName) {
ObjectApi objectApi = swiftApi.getObjectApi(region, containerName);
objectApi.delete(objectName);
out.println("Deleted: " + objectName);
}
// step-10
private Container getContainer(String containerName) {
ContainerApi containerApi = swiftApi.getContainerApi(region);
// ensure container exists
containerApi.create(containerName);
return containerApi.get(containerName);
}
// step-11
static class Fractal {
// only included elements we want to work with
String uuid;
}
static class Fractals {
// only included elements we want to work with
List<Fractal> objects;
}
private void backupFractals(String containerName, String fractalsIp) {
// just need to make sure that there is container
getContainer(containerName);
try {
String response = "";
String endpoint = "http://" + fractalsIp + "/v1/fractal";
URLConnection connection = new URL(endpoint).openConnection();
connection.setRequestProperty("'results_per_page", "-1");
connection.getInputStream();
try (BufferedReader in = new BufferedReader(new InputStreamReader(
connection.getInputStream()))) {
String inputLine;
while ((inputLine = in.readLine()) != null) {
response = response + inputLine;
}
}
Gson gson = new Gson();
Fractals fractals = gson.fromJson(response, Fractals.class);
ObjectApi objectApi = swiftApi.getObjectApi(region, containerName);
fractals.objects.forEach(fractal -> {
try {
String fractalEndpoint = "http://" + fractalsIp + "/fractal/" + fractal.uuid;
URLConnection conn = new URL(fractalEndpoint).openConnection();
try (InputStream inputStream = conn.getInputStream()) {
Payload payload = newInputStreamPayload(inputStream);
String eTag = objectApi.put(fractal.uuid, payload);
out.println(String.format("Backed up %s eTag = %s", fractal.uuid, eTag));
}
} catch (IOException e) {
out.println("Could not backup " + fractal.uuid + "! Cause: " + e.getMessage());
}
});
out.println("Backed up:");
objectApi.list().forEach(object -> out.println(" " + object));
} catch (IOException e) {
out.println("Could not backup fractals! Cause: " + e.getMessage());
}
}
// step-12
private boolean deleteContainer(String containerName) {
ObjectApi objectApi = swiftApi.getObjectApi(region, containerName);
objectApi.list().forEach(object -> objectApi.delete(object.getName()));
ContainerApi containerApi = swiftApi.getContainerApi(region);
return containerApi.deleteIfEmpty(containerName);
}
// step-13
private void createWithMetadata(String containerName, String objectName, String filePath) {
ContainerApi containerApi = swiftApi.getContainerApi(region);
CreateContainerOptions options = CreateContainerOptions.Builder
.metadata(ImmutableMap.of("photos", "of fractals"));
if (containerApi.create(containerName, options)) {
out.println("Uploading: " + objectName);
ObjectApi objectApi = swiftApi.getObjectApi(region, containerName);
Payload payload = newFilePayload(new File(filePath));
PutOptions putOptions = PutOptions.Builder
.metadata(ImmutableMap.of(
"description", "a funny goat",
"created", "2015-06-02"));
String eTag = objectApi.put(objectName, payload, putOptions);
out.println(
String.format("Uploaded %s as \"%s\" eTag = %s", filePath, objectName, eTag));
} else {
out.println("Could not upload " + objectName);
}
}
// step-14
private void uploadLargeFile(String containerName, String pathNameOfLargeFile) {
// Only works with jclouds V2 (in beta at the time of writing). See:
// https://issues.apache.org/jira/browse/JCLOUDS-894
try {
RegionScopedBlobStoreContext context = ContextBuilder.newBuilder(PROVIDER)
.credentials(IDENTITY, PASSWORD)
.endpoint(OS_AUTH_URL)
.buildView(RegionScopedBlobStoreContext.class);
String region = context.getConfiguredRegions().iterator().next();
out.println("Running in region: " + region);
BlobStore blobStore = context.getBlobStore(region);
// create the container if it doesn't exist...
Location location = getOnlyElement(blobStore.listAssignableLocations());
blobStore.createContainerInLocation(location, containerName);
File largeFile = new File(pathNameOfLargeFile);
ByteSource source = Files.asByteSource(largeFile);
Payload payload = Payloads.newByteSourcePayload(source);
payload.getContentMetadata().setContentLength(largeFile.length());
out.println("Uploading file. This may take some time!");
Blob blob = blobStore.blobBuilder(largeFile.getName())
.payload(payload)
.build();
org.jclouds.blobstore.options.PutOptions putOptions =
new org.jclouds.blobstore.options.PutOptions();
String eTag = blobStore.putBlob(containerName, blob, putOptions.multipart());
out.println(String.format("Uploaded %s eTag=%s", largeFile.getName(), eTag));
} catch (UnsupportedOperationException e) {
out.println("Sorry: large file uploads only work in jclouds V2...");
}
}
// step-15
@Override
public void close() throws IOException {
Closeables.close(swiftApi, true);
}
public static void main(String[] args) throws IOException {
try (Durability tester = new Durability()) {
String containerName = "fractals";
String objectName = "an amazing goat";
String goatImageFilePath = "goat.jpg";
String fractalsIp = "IP_API_1";
String pathNameOfLargeFile = "big.img";
tester.createContainer(containerName);
tester.listContainers();
tester.uploadObject(containerName, objectName, goatImageFilePath);
tester.listObjectsInContainer(containerName);
tester.getObjectFromContainer(containerName, objectName);
tester.calculateMd5ForFile(goatImageFilePath);
tester.deleteObject(containerName, objectName);
tester.getContainer(containerName);
tester.backupFractals(containerName, fractalsIp);
tester.deleteContainer(containerName);
tester.createWithMetadata(containerName, objectName, goatImageFilePath);
tester.listContainers();
tester.uploadLargeFile("largeObject", pathNameOfLargeFile);
}
}
}

145
firstapp/source/durability.rst
View File

@ -75,10 +75,12 @@ First, learn how to connect to the Object Storage endpoint:
:start-after: step-1
:end-before: step-2
.. only:: jclouds
.. warning:: This section has not yet been completed for the jclouds SDK.
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-1
:end-before: step-2
.. only:: libcloud
@ -131,6 +133,13 @@ Call yours :code:`fractals`:
TBC
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-2
:end-before: step-3
.. only:: libcloud
.. literalinclude:: ../samples/libcloud/durability.py
@ -158,6 +167,13 @@ all containers in your account:
TBC
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-3
:end-before: step-4
.. only:: libcloud
.. literalinclude:: ../samples/libcloud/durability.py
@ -180,6 +196,13 @@ on line, name it :code:`goat.jpg`, and upload it to your
:start-after: step-4
:end-before: step-5
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-4
:end-before: step-5
.. only:: libcloud
.. literalinclude:: ../samples/libcloud/durability.py
@ -217,6 +240,39 @@ the same:
7513986d3aeb22659079d1bf3dc2468b
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-5
:end-before: step-6
::
Objects in fractals:
SwiftObject{name=an amazing goat,
uri=https://swift.some.org:8888/v1/AUTH_8997868/fractals/an%20amazing%20goat,
etag=439884df9c1c15c59d2cf43008180048,
lastModified=Wed Nov 25 15:09:34 AEDT 2015, metadata={}}
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-6
:end-before: step-7
::
Fetched: an amazing goat
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-7
:end-before: step-8
::
MD5 for file goat.jpg: 439884df9c1c15c59d2cf43008180048
.. only:: libcloud
@ -254,6 +310,13 @@ Finally, clean up by deleting the test object:
:start-after: step-8
:end-before: step-9
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-8
:end-before: step-10
.. only:: libcloud
.. literalinclude:: ../samples/libcloud/durability.py
@ -278,7 +341,7 @@ Back up the Fractals from the database on the Object Storage
Back up the Fractals app images, which are currently stored inside the
database, on Object Storage.
Place the images in the :code:`fractals`' container:
Place the images in the :code:`fractals` container:
.. only:: fog
@ -286,6 +349,13 @@ Place the images in the :code:`fractals`' container:
:start-after: step-10
:end-before: step-11
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-10
:end-before: step-11
.. only:: libcloud
.. literalinclude:: ../samples/libcloud/durability.py
@ -293,7 +363,7 @@ Place the images in the :code:`fractals`' container:
:end-before: step-11
Next, back up all existing fractals from the database to the swift container.
A simple `for` loop takes care of that:
A simple loop takes care of that:
.. note:: Replace :code:`IP_API_1` with the IP address of the API instance.
@ -303,6 +373,13 @@ A simple `for` loop takes care of that:
:start-after: step-11
:end-before: step-12
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-11
:end-before: step-12
.. only:: libcloud
.. literalinclude:: ../samples/libcloud/durability.py
@ -336,8 +413,8 @@ Extra features
Delete containers
~~~~~~~~~~~~~~~~~
To delete a container, make sure that you have removed all objects from the
container before running this script. Otherwise, the script fails:
To delete a container, you must first remove all objects from the container.
Otherwise, the delete operation fails:
.. only:: fog
@ -345,6 +422,13 @@ container before running this script. Otherwise, the script fails:
:start-after: step-12
:end-before: step-13
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-12
:end-before: step-13
.. only:: libcloud
.. literalinclude:: ../samples/libcloud/durability.py
@ -358,19 +442,33 @@ Add metadata to objects
You can complete advanced tasks such as uploading an object with metadata, as
shown in following example. For more information, see the documentation for
your SDK. This option also uses a bit stream to upload the file, iterating bit
by bit over the file and passing those bits to Object Storage as they come.
Compared to loading the entire file in memory and then sending it, this method
is more efficient, especially for larger files.
your SDK.
.. only:: fog
This option also uses a bit stream to upload the file, iterating bit
by bit over the file and passing those bits to Object Storage as they come.
Compared to loading the entire file in memory and then sending it, this method
is more efficient, especially for larger files.
.. literalinclude:: ../samples/fog/durability.rb
:start-after: step-13
:end-before: step-14
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-13
:end-before: step-14
.. only:: libcloud
This option also uses a bit stream to upload the file, iterating bit
by bit over the file and passing those bits to Object Storage as they come.
Compared to loading the entire file in memory and then sending it, this method
is more efficient, especially for larger files.
.. literalinclude:: ../samples/libcloud/durability.py
:start-after: step-13
:end-before: step-14
@ -389,6 +487,20 @@ For efficiency, most Object Storage installations treat large objects,
:start-after: step-14
:end-before: step-15
.. only:: jclouds
If you work with large objects, use the :code:`RegionScopedBlobStoreContext`
class family instead of the ones used so far.
.. note:: Large file uploads that use the :code:`openstack-swift` provider
are supported in only jclouds V2, currently in beta. Also, the
default chunk size is 64 Mb. Consider changing this as homework.
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
:start-after: step-14
:end-before: step-15
.. only:: libcloud
If you work with large objects, use the :code:`ex_multipart_upload_object`
@ -401,6 +513,19 @@ For efficiency, most Object Storage installations treat large objects,
:start-after: step-14
:end-before: step-15
.. only:: jclouds
Complete code sample
~~~~~~~~~~~~~~~~~~~~
This file contains all the code from this tutorial section. This
class lets you view and run the code.
Before you run this class, confirm that you have configured it for
your cloud and the instance running the Fractals application.
.. literalinclude:: ../samples/jclouds/Durability.java
:language: java
Next steps
----------