openstack/api-site
Code Issues Proposed changes

Merge "Added the jclouds scaling out code sample"

Browse Source
This commit is contained in:
Jenkins 2015-11-27 20:59:42 +00:00 committed by Gerrit Code Review
commit 9bb4df652e
4 changed files with 333 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

284
firstapp/samples/jclouds/ScalingOut.java Normal file
View File

@ -0,0 +1,284 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.io.Closeables;
import com.google.inject.Module;
import org.jclouds.ContextBuilder;
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
import org.jclouds.net.domain.IpProtocol;
import org.jclouds.openstack.nova.v2_0.NovaApi;
import org.jclouds.openstack.nova.v2_0.domain.FloatingIP;
import org.jclouds.openstack.nova.v2_0.domain.Ingress;
import org.jclouds.openstack.nova.v2_0.domain.SecurityGroup;
import org.jclouds.openstack.nova.v2_0.domain.ServerCreated;
import org.jclouds.openstack.nova.v2_0.extensions.FloatingIPApi;
import org.jclouds.openstack.nova.v2_0.extensions.SecurityGroupApi;
import org.jclouds.openstack.nova.v2_0.features.ServerApi;
import org.jclouds.openstack.nova.v2_0.options.CreateServerOptions;
import org.jclouds.openstack.nova.v2_0.predicates.ServerPredicates;
import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.Scanner;
import java.util.stream.Collectors;
import static java.lang.System.out;
/**
* A class that shows the jclouds implementation for the scaling out chapter of the
* "Writing your first OpenStack application" book
* (http://developer.openstack.org/firstapp-libcloud/scaling_out.html)
*/
public class ScalingOut implements Closeable {
private final NovaApi novaApi;
private final String region;
private final ServerApi serverApi;
// change the following to fit your OpenStack installation
private static final String KEY_PAIR_NAME = "demokey";
private static final String PROVIDER = "openstack-nova";
private static final String OS_AUTH_URL = "http://controller:5000/v2.0";
// format for identity is tenantName:userName
private static final String IDENTITY = "your_project_name_or_id:your_auth_username";
private static final String IMAGE_ID = "2cccbea0-cea9-4f86-a3ed-065c652adda5";
private static final String FLAVOR_ID = "2";
public ScalingOut(final String password) {
Iterable<Module> modules = ImmutableSet.<Module>of(new SLF4JLoggingModule());
novaApi = ContextBuilder.newBuilder(PROVIDER)
.endpoint(OS_AUTH_URL)
.credentials(IDENTITY, password)
.modules(modules)
.buildApi(NovaApi.class);
region = novaApi.getConfiguredRegions().iterator().next();
serverApi = novaApi.getServerApi(region);
out.println("Running in region: " + region);
}
// step-1
private void deleteInstances() {
List instances = Arrays.asList(
"all-in-one", "app-worker-1", "app-worker-2", "app-controller");
serverApi.listInDetail().concat().forEach(instance -> {
if (instances.contains(instance.getName())) {
out.println("Destroying Instance: " + instance.getName());
serverApi.delete(instance.getId());
}
});
}
private void deleteSecurityGroups() {
List securityGroups = Arrays.asList(
"all-in-one", "control", "worker", "api", "services");
if (novaApi.getSecurityGroupApi(region).isPresent()) {
SecurityGroupApi securityGroupApi = novaApi.getSecurityGroupApi(region).get();
securityGroupApi.list().forEach(securityGroup -> {
if (securityGroups.contains(securityGroup.getName())) {
out.println("Deleting Security Group: " + securityGroup.getName());
securityGroupApi.delete(securityGroup.getId());
}
});
} else {
out.println("No security group extension present; skipping security group delete.");
}
}
// step-2
private Ingress getIngress(int port) {
return Ingress
.builder()
.ipProtocol(IpProtocol.TCP)
.fromPort(port)
.toPort(port)
.build();
}
private void createSecurityGroups() {
if (novaApi.getSecurityGroupApi(region).isPresent()) {
SecurityGroupApi securityGroupApi = novaApi.getSecurityGroupApi(region).get();
SecurityGroup apiGroup = securityGroupApi.createWithDescription("api",
"for API services only");
ImmutableSet.of(22, 80).forEach(port ->
securityGroupApi.createRuleAllowingCidrBlock(
apiGroup.getId(), getIngress(port), "0.0.0.0/0"));
SecurityGroup workerGroup = securityGroupApi.createWithDescription("worker",
"for services that run on a worker node");
securityGroupApi.createRuleAllowingCidrBlock(
workerGroup.getId(), getIngress(22), "0.0.0.0/0");
SecurityGroup controllerGroup = securityGroupApi.createWithDescription("control",
"for services that run on a control node");
ImmutableSet.of(22, 80).forEach(port ->
securityGroupApi.createRuleAllowingCidrBlock(
controllerGroup.getId(), getIngress(port), "0.0.0.0/0"));
securityGroupApi.createRuleAllowingSecurityGroupId(
controllerGroup.getId(), getIngress(5672), workerGroup.getId());
SecurityGroup servicesGroup = securityGroupApi.createWithDescription("services",
"for DB and AMQP services only");
securityGroupApi.createRuleAllowingCidrBlock(
servicesGroup.getId(), getIngress(22), "0.0.0.0/0");
securityGroupApi.createRuleAllowingSecurityGroupId(
servicesGroup.getId(), getIngress(3306), apiGroup.getId());
securityGroupApi.createRuleAllowingSecurityGroupId(
servicesGroup.getId(), getIngress(5672), workerGroup.getId());
securityGroupApi.createRuleAllowingSecurityGroupId(
servicesGroup.getId(), getIngress(5672), apiGroup.getId());
} else {
out.println("No security group extension present; skipping security group create.");
}
}
// step-3
private Optional<FloatingIP> getOrCreateFloatingIP() {
FloatingIP unusedFloatingIP = null;
if (novaApi.getFloatingIPApi(region).isPresent()) {
FloatingIPApi floatingIPApi = novaApi.getFloatingIPApi(region).get();
List<FloatingIP> freeIP = floatingIPApi.list().toList().stream()
.filter(floatingIP1 -> floatingIP1.getInstanceId() == null)
.collect(Collectors.toList());
unusedFloatingIP = freeIP.size() > 0 ? freeIP.get(0) : floatingIPApi.create();
} else {
out.println("No floating ip extension present; skipping floating ip creation.");
}
return Optional.ofNullable(unusedFloatingIP);
}
// step-4
/**
* A helper function to create an instance
*
* @param name The name of the instance that is to be created
* @param options Keypairs, security groups etc...
* @return the id of the newly created instance.
*/
private String createInstance(String name, CreateServerOptions... options) {
out.println("Creating server " + name);
ServerCreated serverCreated = serverApi.create(name, IMAGE_ID, FLAVOR_ID, options);
String id = serverCreated.getId();
ServerPredicates.awaitActive(serverApi).apply(id);
return id;
}
/**
* @return the id of the newly created instance.
*/
private String createAppServicesInstance() {
String userData = "#!/usr/bin/env bash\n" +
"curl -L -s http://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \\\n" +
"-i database -i messaging\n";
CreateServerOptions options = CreateServerOptions.Builder
.keyPairName(KEY_PAIR_NAME)
.securityGroupNames("services")
.userData(userData.getBytes());
return createInstance("app-services", options);
}
// step-5
/**
* @return the id of the newly created instance.
*/
private String createApiInstance(String name, String servicesIp) {
String userData = String.format("#!/usr/bin/env bash\n" +
"curl -L -s http://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \\\n" +
" -i faafo -r api -m 'amqp://guest:guest@%1$s:5672/' \\\n" +
" -d 'mysql+pymysql://faafo:password@%1$s:3306/faafo'", servicesIp);
CreateServerOptions options = CreateServerOptions.Builder
.keyPairName(KEY_PAIR_NAME)
.securityGroupNames("api")
.userData(userData.getBytes());
return createInstance(name, options);
}
/**
* @return the id's of the newly created instances.
*/
private String[] createApiInstances(String servicesIp) {
return new String[]{
createApiInstance("app-api-1", servicesIp),
createApiInstance("app-api-2", servicesIp)
};
}
// step-6
/**
* @return the id of the newly created instance.
*/
private String createWorkerInstance(String name, String apiIp, String servicesIp) {
String userData = String.format("#!/usr/bin/env bash\n" +
"curl -L -s http://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh | bash -s -- \\\n" +
" -i faafo -r worker -e 'http://%s' -m 'amqp://guest:guest@%s:5672/'",
apiIp, servicesIp);
CreateServerOptions options = CreateServerOptions.Builder
.keyPairName(KEY_PAIR_NAME)
.securityGroupNames("worker")
.userData(userData.getBytes());
return createInstance(name, options);
}
private void createWorkerInstances(String apiIp, String servicesIp) {
createWorkerInstance("app-worker-1", apiIp, servicesIp);
createWorkerInstance("app-worker-2", apiIp, servicesIp);
createWorkerInstance("app-worker-3", apiIp, servicesIp);
}
// step-7
private String getPublicIp(String serverId) {
String publicIP = serverApi.get(serverId).getAccessIPv4();
if (publicIP == null) {
Optional<FloatingIP> optionalFloatingIP = getOrCreateFloatingIP();
if (optionalFloatingIP.isPresent()) {
publicIP = optionalFloatingIP.get().getIp();
novaApi.getFloatingIPApi(region).get().addToServer(publicIP, serverId);
}
}
return publicIP;
}
private String getPublicOrPrivateIP(String serverId) {
String result = serverApi.get(serverId).getAccessIPv4();
if (result == null) {
// then there must be private one...
result = serverApi.get(serverId).getAddresses().values().iterator().next().getAddr();
}
return result;
}
private void setupFaafo() {
deleteInstances();
deleteSecurityGroups();
createSecurityGroups();
String serviceId = createAppServicesInstance();
String servicesIp = getPublicOrPrivateIP(serviceId);
String[] apiIds = createApiInstances(servicesIp);
String apiIp = getPublicIp(apiIds[0]);
createWorkerInstances(apiIp, servicesIp);
out.println("The Fractals app will be deployed to http://" + apiIp);
}
@Override
public void close() throws IOException {
Closeables.close(novaApi, true);
}
public static void main(String... args) throws IOException {
try (Scanner scanner = new Scanner(System.in)) {
System.out.println("Please enter your password: ");
String password = scanner.next();
try (ScalingOut gs = new ScalingOut(password)) {
gs.setupFaafo();
}
}
}
}

48
firstapp/source/scaling_out.rst
View File

@ -167,6 +167,13 @@ cloud are no longer working, remove them and re-create something new.
:start-after: step-1
:end-before: step-2
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-1
:end-before: step-2
Extra security groups
---------------------
@ -193,6 +200,13 @@ groups.
:start-after: step-2
:end-before: step-3
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-2
:end-before: step-3
A floating IP helper function
-----------------------------
@ -218,6 +232,13 @@ floating IP quota too quickly.
:start-after: step-3
:end-before: step-4
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-3
:end-before: step-4
Split the database and message queue
------------------------------------
@ -244,6 +265,13 @@ fractals and to coordinate the communication between the services.
:start-after: step-4
:end-before: step-5
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-4
:end-before: step-5
Scale the API service
---------------------
@ -273,6 +301,13 @@ multiple API services:
:start-after: step-5
:end-before: step-6
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-5
:end-before: step-6
These services are client-facing, so unlike the workers they do not
use a message queue to distribute tasks. Instead, you must introduce
some kind of load balancing mechanism to share incoming requests
@ -284,7 +319,6 @@ a `DNS round robin <http://en.wikipedia.org/wiki/Round- robin_DNS>`_
to do that automatically. However, OpenStack networking can provide
Load Balancing as a Service, which :doc:`/networking` explains.
.. todo:: Add a note that we demonstrate this by using the first API
instance for the workers and the second API instance for the
load simulation.
@ -313,6 +347,13 @@ To increase the overall capacity, add three workers:
:start-after: step-6
:end-before: step-7
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java
:start-after: step-6
:end-before: step-7
Adding this capacity enables you to deal with a higher number of
requests for fractals. As soon as these worker instances start, they
begin checking the message queue for requests, reducing the overall
@ -477,3 +518,8 @@ authentication information, the flavor ID, and image ID.
.. literalinclude:: ../samples/libcloud/scaling_out.py
:language: python
.. only:: jclouds
.. literalinclude:: ../samples/jclouds/ScalingOut.java
:language: java

2
tools/build-firstapp-rst.sh
View File

@ -9,7 +9,7 @@ for tag in libcloud; do
done
# Draft documents
for tag in dotnet fog openstacksdk pkgcloud shade; do
for tag in dotnet fog openstacksdk pkgcloud shade jclouds; do
tools/build-rst.sh firstapp \
--tag ${tag} --target "api-ref/draft/firstapp-${tag}"
done

2
tools/generatepot-rst.sh
View File

@ -51,7 +51,7 @@ if [ ${DOCNAME} = "install-guide" ] ; then
TAG="-t obs -t rdo -t ubuntu -t debian"
fi
if [ ${DOCNAME} = "firstapp" ] ; then
TAG="-t libcloud -t dotnet -t fog -t openstacksdk -t pkgcloud -t shade"
TAG="-t libcloud -t dotnet -t fog -t openstacksdk -t pkgcloud -t shade -t jclouds"
fi
sphinx-build -b gettext $TAG ${DIRECTORY}/source/ \
${DIRECTORY}/source/locale/