openstack/api-site
Code Issues Proposed changes

Added the jclouds scaling out code sample

Browse Source 
Added a scaling out code sample for jclouds and modified the
scaling out chapter to reference it.

Also modified the build so that the jclouds draft documentation
is now generated so that we can see the code appear in context.

Change-Id: Iaf9ee952e008bdb68e52717524de723597da0690
Partial-Bug: #1449330
This commit is contained in:
Martin Paulo 2015-11-23 15:50:12 +11:00
parent f71e9fca15
commit 7682d59aa9
4 changed files with 333 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

284
firstapp/samples/jclouds/ScalingOut.java Normal file
View File

@ -0,0 +1,284 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.io.Closeables;
import com.google.inject.Module;
import org.jclouds.ContextBuilder;
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
import org.jclouds.net.domain.IpProtocol;
import org.jclouds.openstack.nova.v2_0.NovaApi;
import org.jclouds.openstack.nova.v2_0.domain.FloatingIP;
import org.jclouds.openstack.nova.v2_0.domain.Ingress;
import org.jclouds.openstack.nova.v2_0.domain.SecurityGroup;
import org.jclouds.openstack.nova.v2_0.domain.ServerCreated;
import org.jclouds.openstack.nova.v2_0.extensions.FloatingIPApi;
import org.jclouds.openstack.nova.v2_0.extensions.SecurityGroupApi;
import org.jclouds.openstack.nova.v2_0.features.ServerApi;
import org.jclouds.openstack.nova.v2_0.options.CreateServerOptions;
import org.jclouds.openstack.nova.v2_0.predicates.ServerPredicates;
import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.Scanner;
import java.util.stream.Collectors;
import static java.lang.System.out;
/**
* A class that shows the jclouds implementation for the scaling out chapter of the
* "Writing your first OpenStack application" book
* (http://developer.openstack.org/firstapp-libcloud/scaling_out.html)
*/
public class ScalingOut implements Closeable {
private final NovaApi novaApi;
private final String region;
private final ServerApi serverApi;
// change the following to fit your OpenStack installation
private static final String KEY_PAIR_NAME = "demokey";
private static final String PROVIDER = "openstack-nova";
private static final String OS_AUTH_URL = "http://controller:5000/v2.0";
// format for identity is tenantName:userName
private static final String IDENTITY = "your_project_name_or_id:your_auth_username";
private static final String IMAGE_ID = "2cccbea0-cea9-4f86-a3ed-065c652adda5";
private static final String FLAVOR_ID = "2";
public ScalingOut(final String password) {
Iterable<Module> modules = ImmutableSet.<Module>of(new SLF4JLoggingModule());
novaApi = ContextBuilder.newBuilder(PROVIDER)
.endpoint(OS_AUTH_URL)
.credentials(IDENTITY, password)
.modules(modules)
.buildApi(NovaApi.class);
region = novaApi.getConfiguredRegions().iterator().next();
serverApi = novaApi.getServerApi(region);
out.println("Running in region: " + region);
}
// step-1
private void deleteInstances() {
List instances = Arrays.asList(
"all-in-one", "app-worker-1", "app-worker-2", "app-controller");
serverApi.listInDetail().concat().forEach(instance -> {
if (instances.contains(instance.getName())) {
out.println("Destroying Instance: " + instance.getName());
serverApi.delete(instance.getId());
}
});
}
private void deleteSecurityGroups() {
List securityGroups = Arrays.asList(
"all-in-one", "control", "worker", "api", "services");
if (novaApi.getSecurityGroupApi(region).isPresent()) {
SecurityGroupApi securityGroupApi = novaApi.getSecurityGroupApi(region).get();
securityGroupApi.list().forEach(securityGroup -> {
if (securityGroups.contains(securityGroup.getName())) {
out.println("Deleting Security Group: " + securityGroup.getName());
securityGroupApi.delete(securityGroup.getId());
}
});
} else {
out.println("No security group extension present; skipping security group delete.");
}
}
// step-2
private Ingress getIngress(int port) {
return Ingress
.builder()
.ipProtocol(IpProtocol.TCP)
.fromPort(port)
.toPort(port)
.build();
}
private void createSecurityGroups() {
if (novaApi.getSecurityGroupApi(region).isPresent()) {
SecurityGroupApi securityGroupApi = novaApi.getSecurityGroupApi(region).get();
SecurityGroup apiGroup = securityGroupApi.createWithDescription("api",
"for API services only");
ImmutableSet.of(22, 80).forEach(port ->
securityGroupApi.createRuleAllowingCidrBlock(
apiGroup.getId(), getIngress(port), "0.0.0.0/0"));
SecurityGroup workerGroup = securityGroupApi.createWithDescription("worker",
"for services that run on a worker node");
securityGroupApi.createRuleAllowingCidrBlock(
workerGroup.getId(), getIngress(22), "0.0.0.0/0");
SecurityGroup controllerGroup = securityGroupApi.createWithDescription("control",
"for services that run on a control node");
ImmutableSet.of(22, 80).forEach(port ->
securityGroupApi.createRuleAllowingCidrBlock(
controllerGroup.getId(), getIngress(port), "0.0.0.0/0"));
securityGroupApi.createRuleAllowingSecurityGroupId(
controllerGroup.getId(), getIngress(5672), workerGroup.getId());
SecurityGroup servicesGroup = securityGroupApi.createWithDescription("services",
"for DB and AMQP services only");
securityGroupApi.createRuleAllowingCidrBlock(
servicesGroup.getId(), getIngress(22), "0.0.0.0/0");
securityGroupApi.createRuleAllowingSecurityGroupId(
servicesGroup.getId(), getIngress(3306), apiGroup.getId());
securityGroupApi.createRuleAllowingSecurityGroupId(
servicesGroup.getId(), getIngress(5672), workerGroup.getId());
securityGroupApi.createRuleAllowingSecurityGroupId(
servicesGroup.getId(), getIngress(5672), apiGroup.getId());
} else {
out.println("No security group extension present; skipping security group create.");
}
}
// step-3
private Optional<FloatingIP> getOrCreateFloatingIP() {
FloatingIP unusedFloatingIP = null;
if (novaApi.getFloatingIPApi(region).isPresent()) {
FloatingIPApi floatingIPApi = novaApi.getFloatingIPApi(region).get();
List<FloatingIP> freeIP = floatingIPApi.list().toList().stream()
.filter(floatingIP1 -> floatingIP1.getInstanceId() == null)
.collect(Collectors.toList());
unusedFloatingIP = freeIP.size() > 0 ? freeIP.get(0) : floatingIPApi.create();
} else {
out.println("No floating ip extension present; skipping floating ip creation.");
}
return Optional.ofNullable(unusedFloatingIP);
}
// step-4
/**
* A helper function to create an instance
*
* @param name The name of the instance that is to be created
* @param options Keypairs, security groups etc...
* @return the id of the newly created instance.
*/
private String createInstance(String name, CreateServerOptions... options) {
out.println("Creating server " + name);
ServerCreated serverCreated = serverApi.create(name, IMAGE_ID, FLAVOR_ID, options);
String id = serverCreated.getId();
ServerPredicates.awaitActive(serverApi).apply(id);
return id;
}
/**
* @return the id of the newly created instance.
*/
private String createAppServicesInstance() {
String userData = "#!/usr/bin/env bash\n" +
"curl -L -s http://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \\\n" +
"-i database -i messaging\n";
CreateServerOptions options = CreateServerOptions.Builder
.keyPairName(KEY_PAIR_NAME)
.securityGroupNames("services")
.userData(userData.getBytes());
return createInstance("app-services", options);
}
// step-5
/**
* @return the id of the newly created instance.
*/
private String createApiInstance(String name, String servicesIp) {
String userData = String.format("#!/usr/bin/env bash\n" +
"curl -L -s http://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \\\n" +
" -i faafo -r api -m 'amqp://guest:guest@%1$s:5672/' \\\n" +
" -d 'mysql+pymysql://faafo:password@%1$s:3306/faafo'", servicesIp);
CreateServerOptions options = CreateServerOptions.Builder
.keyPairName(KEY_PAIR_NAME)
.securityGroupNames("api")
.userData(userData.getBytes());
return createInstance(name, options);
}
/**
* @return the id's of the newly created instances.
*/
private String[] createApiInstances(String servicesIp) {
return new String[]{
createApiInstance("app-api-1", servicesIp),
createApiInstance("app-api-2", servicesIp)
};
}
// step-6
/**
* @return the id of the newly created instance.
*/
private String createWorkerInstance(String name, String apiIp, String servicesIp) {
String userData = String.format("#!/usr/bin/env bash\n" +
"curl -L -s http://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \\\n" +
" -i faafo -r worker -e 'http://%s' -m 'amqp://guest:guest@%s:5672/'",
apiIp, servicesIp);
CreateServerOptions options = CreateServerOptions.Builder
.keyPairName(KEY_PAIR_NAME)
.securityGroupNames("worker")
.userData(userData.getBytes());
return createInstance(name, options);
}
private void createWorkerInstances(String apiIp, String servicesIp) {
createWorkerInstance("app-worker-1", apiIp, servicesIp);
createWorkerInstance("app-worker-2", apiIp, servicesIp);
createWorkerInstance("app-worker-3", apiIp, servicesIp);
}
// step-7
private String getPublicIp(String serverId) {
String publicIP = serverApi.get(serverId).getAccessIPv4();
if (publicIP == null) {
Optional<FloatingIP> optionalFloatingIP = getOrCreateFloatingIP();
if (optionalFloatingIP.isPresent()) {
publicIP = optionalFloatingIP.get().getIp();
novaApi.getFloatingIPApi(region).get().addToServer(publicIP, serverId);
}
}
return publicIP;
}
private String getPublicOrPrivateIP(String serverId) {
String result = serverApi.get(serverId).getAccessIPv4();
if (result == null) {
// then there must be private one...
result = serverApi.get(serverId).getAddresses().values().iterator().next().getAddr();
}
return result;
}
private void setupFaafo() {
deleteInstances();
deleteSecurityGroups();
createSecurityGroups();
String serviceId = createAppServicesInstance();
String servicesIp = getPublicOrPrivateIP(serviceId);
String[] apiIds = createApiInstances(servicesIp);
String apiIp = getPublicIp(apiIds[0]);
createWorkerInstances(apiIp, servicesIp);
out.println("The Fractals app will be deployed to http://" + apiIp);
}
@Override
public void close() throws IOException {
Closeables.close(novaApi, true);
}
public static void main(String... args) throws IOException {
try (Scanner scanner = new Scanner(System.in)) {
System.out.println("Please enter your password: ");
String password = scanner.next();
try (ScalingOut gs = new ScalingOut(password)) {
gs.setupFaafo();
}
}
}
}

48
firstapp/source/scaling_out.rst
View File

@ -167,6 +167,13 @@ cloud are no longer working, remove them and re-create something new.
:start-after: step-1
:end-before: step-2
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-1
:end-before: step-2
Extra security groups
---------------------
@ -193,6 +200,13 @@ groups.
:start-after: step-2
:end-before: step-3
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-2
:end-before: step-3
A floating IP helper function
-----------------------------
@ -218,6 +232,13 @@ floating IP quota too quickly.
:start-after: step-3
:end-before: step-4
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-3
:end-before: step-4
Split the database and message queue
------------------------------------
@ -244,6 +265,13 @@ fractals and to coordinate the communication between the services.
:start-after: step-4
:end-before: step-5
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-4
:end-before: step-5
Scale the API service
---------------------
@ -273,6 +301,13 @@ multiple API services:
:start-after: step-5
:end-before: step-6
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-5
:end-before: step-6
These services are client-facing, so unlike the workers they do not
use a message queue to distribute tasks. Instead, you must introduce
some kind of load balancing mechanism to share incoming requests
@ -284,7 +319,6 @@ a `DNS round robin <http://en.wikipedia.org/wiki/Round- robin_DNS>`_
to do that automatically. However, OpenStack networking can provide
Load Balancing as a Service, which :doc:`/networking` explains.
.. todo:: Add a note that we demonstrate this by using the first API
instance for the workers and the second API instance for the
load simulation.
@ -313,6 +347,13 @@ To increase the overall capacity, add three workers:
:start-after: step-6
:end-before: step-7
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-6
:end-before: step-7
Adding this capacity enables you to deal with a higher number of
requests for fractals. As soon as these worker instances start, they
begin checking the message queue for requests, reducing the overall
@ -477,3 +518,8 @@ authentication information, the flavor ID, and image ID.
.. literalinclude:: ../samples/libcloud/scaling_out.py
:language: python
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java

2
tools/build-firstapp-rst.sh
View File

@ -9,7 +9,7 @@ for tag in libcloud; do
done
# Draft documents
for tag in dotnet fog openstacksdk pkgcloud shade; do
for tag in dotnet fog openstacksdk pkgcloud shade jclouds; do
tools/build-rst.sh firstapp \
--tag ${tag} --target "api-ref/draft/firstapp-${tag}"
done

2
tools/generatepot-rst.sh
View File

@ -51,7 +51,7 @@ if [ ${DOCNAME} = "install-guide" ] ; then
TAG="-t obs -t rdo -t ubuntu -t debian"
fi
if [ ${DOCNAME} = "firstapp" ] ; then
TAG="-t libcloud -t dotnet -t fog -t openstacksdk -t pkgcloud -t shade"
TAG="-t libcloud -t dotnet -t fog -t openstacksdk -t pkgcloud -t shade -t jclouds"
fi
sphinx-build -b gettext $TAG ${DIRECTORY}/source/ \
${DIRECTORY}/source/locale/