Added the jclouds scaling out code sample

Added a scaling out code sample for jclouds and modified the
scaling out chapter to reference it.

Also modified the build so that the jclouds draft documentation
is now generated so that we can see the code appear in context.

Change-Id: Iaf9ee952e008bdb68e52717524de723597da0690
Partial-Bug: #1449330

firstapp/samples/jclouds/ScalingOut.java

@@ -0,0 +1,284 @@
+import com.google.common.collect.ImmutableSet;
+import com.google.common.io.Closeables;
+import com.google.inject.Module;
+import org.jclouds.ContextBuilder;
+import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
+import org.jclouds.net.domain.IpProtocol;
+import org.jclouds.openstack.nova.v2_0.NovaApi;
+import org.jclouds.openstack.nova.v2_0.domain.FloatingIP;
+import org.jclouds.openstack.nova.v2_0.domain.Ingress;
+import org.jclouds.openstack.nova.v2_0.domain.SecurityGroup;
+import org.jclouds.openstack.nova.v2_0.domain.ServerCreated;
+import org.jclouds.openstack.nova.v2_0.extensions.FloatingIPApi;
+import org.jclouds.openstack.nova.v2_0.extensions.SecurityGroupApi;
+import org.jclouds.openstack.nova.v2_0.features.ServerApi;
+import org.jclouds.openstack.nova.v2_0.options.CreateServerOptions;
+import org.jclouds.openstack.nova.v2_0.predicates.ServerPredicates;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.Scanner;
+import java.util.stream.Collectors;
+
+import static java.lang.System.out;
+
+/**
+ * A class that shows the jclouds implementation for the scaling out chapter of the
+ * "Writing your first OpenStack application" book
+ * (http://developer.openstack.org/firstapp-libcloud/scaling_out.html)
+ */
+public class ScalingOut implements Closeable {
+
+
+    private final NovaApi novaApi;
+    private final String region;
+    private final ServerApi serverApi;
+
+    // change the following to fit your OpenStack installation
+    private static final String KEY_PAIR_NAME = "demokey";
+    private static final String PROVIDER = "openstack-nova";
+    private static final String OS_AUTH_URL = "http://controller:5000/v2.0";
+    // format for identity is tenantName:userName
+    private static final String IDENTITY = "your_project_name_or_id:your_auth_username";
+    private static final String IMAGE_ID = "2cccbea0-cea9-4f86-a3ed-065c652adda5";
+    private static final String FLAVOR_ID = "2";
+
+    public ScalingOut(final String password) {
+        Iterable<Module> modules = ImmutableSet.<Module>of(new SLF4JLoggingModule());
+        novaApi = ContextBuilder.newBuilder(PROVIDER)
+                .endpoint(OS_AUTH_URL)
+                .credentials(IDENTITY, password)
+                .modules(modules)
+                .buildApi(NovaApi.class);
+        region = novaApi.getConfiguredRegions().iterator().next();
+        serverApi = novaApi.getServerApi(region);
+        out.println("Running in region: " + region);
+    }
+
+// step-1
+
+    private void deleteInstances() {
+        List instances = Arrays.asList(
+                "all-in-one", "app-worker-1", "app-worker-2", "app-controller");
+        serverApi.listInDetail().concat().forEach(instance -> {
+            if (instances.contains(instance.getName())) {
+                out.println("Destroying Instance: " + instance.getName());
+                serverApi.delete(instance.getId());
+            }
+        });
+    }
+
+    private void deleteSecurityGroups() {
+        List securityGroups = Arrays.asList(
+                "all-in-one", "control", "worker", "api", "services");
+        if (novaApi.getSecurityGroupApi(region).isPresent()) {
+            SecurityGroupApi securityGroupApi = novaApi.getSecurityGroupApi(region).get();
+            securityGroupApi.list().forEach(securityGroup -> {
+                if (securityGroups.contains(securityGroup.getName())) {
+                    out.println("Deleting Security Group: " + securityGroup.getName());
+                    securityGroupApi.delete(securityGroup.getId());
+                }
+            });
+        } else {
+            out.println("No security group extension present; skipping security group delete.");
+        }
+    }
+
+// step-2
+
+    private Ingress getIngress(int port) {
+        return Ingress
+                .builder()
+                .ipProtocol(IpProtocol.TCP)
+                .fromPort(port)
+                .toPort(port)
+                .build();
+    }
+
+    private void createSecurityGroups() {
+        if (novaApi.getSecurityGroupApi(region).isPresent()) {
+            SecurityGroupApi securityGroupApi = novaApi.getSecurityGroupApi(region).get();
+            SecurityGroup apiGroup = securityGroupApi.createWithDescription("api",
+                    "for API services only");
+            ImmutableSet.of(22, 80).forEach(port ->
+                    securityGroupApi.createRuleAllowingCidrBlock(
+                            apiGroup.getId(), getIngress(port), "0.0.0.0/0"));
+
+            SecurityGroup workerGroup = securityGroupApi.createWithDescription("worker",
+                    "for services that run on a worker node");
+            securityGroupApi.createRuleAllowingCidrBlock(
+                    workerGroup.getId(), getIngress(22), "0.0.0.0/0");
+
+            SecurityGroup controllerGroup = securityGroupApi.createWithDescription("control",
+                    "for services that run on a control node");
+            ImmutableSet.of(22, 80).forEach(port ->
+                    securityGroupApi.createRuleAllowingCidrBlock(
+                            controllerGroup.getId(), getIngress(port), "0.0.0.0/0"));
+            securityGroupApi.createRuleAllowingSecurityGroupId(
+                    controllerGroup.getId(), getIngress(5672), workerGroup.getId());
+
+            SecurityGroup servicesGroup = securityGroupApi.createWithDescription("services",
+                    "for DB and AMQP services only");
+            securityGroupApi.createRuleAllowingCidrBlock(
+                    servicesGroup.getId(), getIngress(22), "0.0.0.0/0");
+            securityGroupApi.createRuleAllowingSecurityGroupId(
+                    servicesGroup.getId(), getIngress(3306), apiGroup.getId());
+            securityGroupApi.createRuleAllowingSecurityGroupId(
+                    servicesGroup.getId(), getIngress(5672), workerGroup.getId());
+            securityGroupApi.createRuleAllowingSecurityGroupId(
+                    servicesGroup.getId(), getIngress(5672), apiGroup.getId());
+        } else {
+            out.println("No security group extension present; skipping security group create.");
+        }
+    }
+
+// step-3
+
+    private Optional<FloatingIP> getOrCreateFloatingIP() {
+        FloatingIP unusedFloatingIP = null;
+        if (novaApi.getFloatingIPApi(region).isPresent()) {
+            FloatingIPApi floatingIPApi = novaApi.getFloatingIPApi(region).get();
+            List<FloatingIP> freeIP = floatingIPApi.list().toList().stream()
+                    .filter(floatingIP1 -> floatingIP1.getInstanceId() == null)
+                    .collect(Collectors.toList());
+            unusedFloatingIP = freeIP.size() > 0 ? freeIP.get(0) : floatingIPApi.create();
+        } else {
+            out.println("No floating ip extension present; skipping floating ip creation.");
+        }
+        return Optional.ofNullable(unusedFloatingIP);
+    }
+
+// step-4
+
+    /**
+     * A helper function to create an instance
+     *
+     * @param name    The name of the instance that is to be created
+     * @param options Keypairs, security groups etc...
+     * @return the id of the newly created instance.
+     */
+    private String createInstance(String name, CreateServerOptions... options) {
+        out.println("Creating server " + name);
+        ServerCreated serverCreated = serverApi.create(name, IMAGE_ID, FLAVOR_ID, options);
+        String id = serverCreated.getId();
+        ServerPredicates.awaitActive(serverApi).apply(id);
+        return id;
+    }
+
+    /**
+     * @return the id of the newly created instance.
+     */
+    private String createAppServicesInstance() {
+        String userData = "#!/usr/bin/env bash\n" +
+                "curl -L -s http://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \\\n" +
+                "-i database -i messaging\n";
+        CreateServerOptions options = CreateServerOptions.Builder
+                .keyPairName(KEY_PAIR_NAME)
+                .securityGroupNames("services")
+                .userData(userData.getBytes());
+        return createInstance("app-services", options);
+    }
+
+// step-5
+
+    /**
+     * @return the id of the newly created instance.
+     */
+    private String createApiInstance(String name, String servicesIp) {
+        String userData = String.format("#!/usr/bin/env bash\n" +
+                "curl -L -s http://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \\\n" +
+                "    -i faafo -r api -m 'amqp://guest:guest@%1$s:5672/' \\\n" +
+                "    -d 'mysql+pymysql://faafo:password@%1$s:3306/faafo'", servicesIp);
+        CreateServerOptions options = CreateServerOptions.Builder
+                .keyPairName(KEY_PAIR_NAME)
+                .securityGroupNames("api")
+                .userData(userData.getBytes());
+        return createInstance(name, options);
+    }
+
+    /**
+     * @return the id's of the newly created instances.
+     */
+    private String[] createApiInstances(String servicesIp) {
+        return new String[]{
+                createApiInstance("app-api-1", servicesIp),
+                createApiInstance("app-api-2", servicesIp)
+        };
+    }
+
+// step-6
+
+    /**
+     * @return the id of the newly created instance.
+     */
+    private String createWorkerInstance(String name, String apiIp, String servicesIp) {
+        String userData = String.format("#!/usr/bin/env bash\n" +
+                "curl -L -s http://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \\\n" +
+                "    -i faafo -r worker -e 'http://%s' -m 'amqp://guest:guest@%s:5672/'",
+                apiIp, servicesIp);
+        CreateServerOptions options = CreateServerOptions.Builder
+                .keyPairName(KEY_PAIR_NAME)
+                .securityGroupNames("worker")
+                .userData(userData.getBytes());
+        return createInstance(name, options);
+    }
+
+    private void createWorkerInstances(String apiIp, String servicesIp) {
+        createWorkerInstance("app-worker-1", apiIp, servicesIp);
+        createWorkerInstance("app-worker-2", apiIp, servicesIp);
+        createWorkerInstance("app-worker-3", apiIp, servicesIp);
+    }
+
+// step-7
+
+    private String getPublicIp(String serverId) {
+        String publicIP = serverApi.get(serverId).getAccessIPv4();
+        if (publicIP == null) {
+            Optional<FloatingIP> optionalFloatingIP = getOrCreateFloatingIP();
+            if (optionalFloatingIP.isPresent()) {
+                publicIP = optionalFloatingIP.get().getIp();
+                novaApi.getFloatingIPApi(region).get().addToServer(publicIP, serverId);
+            }
+        }
+        return publicIP;
+    }
+
+    private String getPublicOrPrivateIP(String serverId) {
+        String result = serverApi.get(serverId).getAccessIPv4();
+        if (result == null) {
+            // then there must be private one...
+            result = serverApi.get(serverId).getAddresses().values().iterator().next().getAddr();
+        }
+        return result;
+    }
+
+    private void setupFaafo() {
+        deleteInstances();
+        deleteSecurityGroups();
+        createSecurityGroups();
+        String serviceId = createAppServicesInstance();
+        String servicesIp = getPublicOrPrivateIP(serviceId);
+        String[] apiIds = createApiInstances(servicesIp);
+        String apiIp = getPublicIp(apiIds[0]);
+        createWorkerInstances(apiIp, servicesIp);
+        out.println("The Fractals app will be deployed to http://" + apiIp);
+    }
+
+    @Override
+    public void close() throws IOException {
+        Closeables.close(novaApi, true);
+    }
+
+    public static void main(String... args) throws IOException {
+        try (Scanner scanner = new Scanner(System.in)) {
+            System.out.println("Please enter your password: ");
+            String password = scanner.next();
+            try (ScalingOut gs = new ScalingOut(password)) {
+                gs.setupFaafo();
+            }
+        }
+    }
+}

firstapp/source/scaling_out.rst

@@ -167,6 +167,13 @@ cloud are no longer working, remove them and re-create something new.
         :start-after: step-1
         :end-before: step-2
 
+.. only:: jclouds
+
+    .. literalinclude:: ../samples/jclouds/ScalingOut.java
+        :language: java
+        :start-after: step-1
+        :end-before: step-2
+
 
 Extra security groups
 ---------------------
@@ -193,6 +200,13 @@ groups.
         :start-after: step-2
         :end-before: step-3
 
+.. only:: jclouds
+
+    .. literalinclude:: ../samples/jclouds/ScalingOut.java
+        :language: java
+        :start-after: step-2
+        :end-before: step-3
+
 A floating IP helper function
 -----------------------------
 
@@ -218,6 +232,13 @@ floating IP quota too quickly.
         :start-after: step-3
         :end-before: step-4
 
+.. only:: jclouds
+
+    .. literalinclude:: ../samples/jclouds/ScalingOut.java
+        :language: java
+        :start-after: step-3
+        :end-before: step-4
+
 Split the database and message queue
 ------------------------------------
 
@@ -244,6 +265,13 @@ fractals and to coordinate the communication between the services.
         :start-after: step-4
         :end-before: step-5
 
+ .. only:: jclouds
+
+    .. literalinclude:: ../samples/jclouds/ScalingOut.java
+        :language: java
+        :start-after: step-4
+        :end-before: step-5
+
 Scale the API service
 ---------------------
 
@@ -273,6 +301,13 @@ multiple API services:
         :start-after: step-5
         :end-before: step-6
 
+ .. only:: jclouds
+
+    .. literalinclude:: ../samples/jclouds/ScalingOut.java
+        :language: java
+        :start-after: step-5
+        :end-before: step-6
+
 These services are client-facing, so unlike the workers they do not
 use a message queue to distribute tasks. Instead, you must introduce
 some kind of load balancing mechanism to share incoming requests
@@ -284,7 +319,6 @@ a `DNS round robin <http://en.wikipedia.org/wiki/Round- robin_DNS>`_
 to do that automatically. However, OpenStack networking can provide
 Load Balancing as a Service, which :doc:`/networking` explains.
 
-
 .. todo:: Add a note that we demonstrate this by using the first API
           instance for the workers and the second API instance for the
           load simulation.
@@ -313,6 +347,13 @@ To increase the overall capacity, add three workers:
         :start-after: step-6
         :end-before: step-7
 
+ .. only:: jclouds
+
+    .. literalinclude:: ../samples/jclouds/ScalingOut.java
+        :language: java
+        :start-after: step-6
+        :end-before: step-7
+
 Adding this capacity enables you to deal with a higher number of
 requests for fractals. As soon as these worker instances start, they
 begin checking the message queue for requests, reducing the overall
@@ -477,3 +518,8 @@ authentication information, the flavor ID, and image ID.
 
     .. literalinclude:: ../samples/libcloud/scaling_out.py
        :language: python
+
+.. only:: jclouds
+
+    .. literalinclude:: ../samples/jclouds/ScalingOut.java
+       :language: java

tools/build-firstapp-rst.sh

@@ -9,7 +9,7 @@ for tag in libcloud; do
 done
 
 # Draft documents
-for tag in dotnet fog openstacksdk pkgcloud shade; do
+for tag in dotnet fog openstacksdk pkgcloud shade jclouds; do
     tools/build-rst.sh firstapp  \
         --tag ${tag} --target "api-ref/draft/firstapp-${tag}"
 done

tools/generatepot-rst.sh

@@ -51,7 +51,7 @@ if [ ${DOCNAME} = "install-guide" ] ; then
     TAG="-t obs -t rdo -t ubuntu -t debian"
 fi
 if [ ${DOCNAME} = "firstapp" ] ; then
-    TAG="-t libcloud  -t dotnet -t fog -t openstacksdk -t pkgcloud -t shade"
+    TAG="-t libcloud  -t dotnet -t fog -t openstacksdk -t pkgcloud -t shade -t jclouds"
 fi
 sphinx-build -b gettext $TAG ${DIRECTORY}/source/ \
     ${DIRECTORY}/source/locale/