From b06266257692588f6d897a004ff2e611470730b3 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 8 Oct 2015 09:41:35 +0200 Subject: [PATCH 1/9] First stage porting torrent transport to new transorts --- resources/transport_torrent/meta.yml | 14 ++++++++++++++ .../transport_torrent/scripts/solar_torrent.py | 1 + 2 files changed, 15 insertions(+) create mode 100644 resources/transport_torrent/meta.yml create mode 120000 resources/transport_torrent/scripts/solar_torrent.py diff --git a/resources/transport_torrent/meta.yml b/resources/transport_torrent/meta.yml new file mode 100644 index 0000000..9eb5627 --- /dev/null +++ b/resources/transport_torrent/meta.yml @@ -0,0 +1,14 @@ +id: transport_torrent +input: + name: + schema: str! + value: torrent + location_id: + schema: str + value: + reverse: True + is_own: False + transports_id: + schema: str + value: + is_emit: False diff --git a/resources/transport_torrent/scripts/solar_torrent.py b/resources/transport_torrent/scripts/solar_torrent.py new file mode 120000 index 0000000..0f59be0 --- /dev/null +++ b/resources/transport_torrent/scripts/solar_torrent.py @@ -0,0 +1 @@ +../../../solar/solar/core/transports/helpers/solar_torrent.py \ No newline at end of file From d3cfdc1f9f27d3f84189c3173c8c50fdeff2d4eb Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 8 Oct 2015 10:28:45 +0200 Subject: [PATCH 2/9] Correct meta.yaml name --- resources/transport_torrent/{meta.yml => meta.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename resources/transport_torrent/{meta.yml => meta.yaml} (100%) diff --git a/resources/transport_torrent/meta.yml b/resources/transport_torrent/meta.yaml similarity index 100% rename from resources/transport_torrent/meta.yml rename to resources/transport_torrent/meta.yaml From dae8b1a98a860a71308fc366b9b47c5d7807675d Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 8 Oct 2015 12:10:33 +0200 Subject: [PATCH 3/9] torrent transport resource improvements --- resources/transport_torrent/meta.yaml | 4 ++++ resources/transports/meta.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/resources/transport_torrent/meta.yaml b/resources/transport_torrent/meta.yaml index 9eb5627..217a544 100644 --- a/resources/transport_torrent/meta.yaml +++ b/resources/transport_torrent/meta.yaml @@ -1,5 +1,9 @@ id: transport_torrent +handler: ansible input: + trackers: + schema: [str!] + value: [] name: schema: str! value: torrent diff --git a/resources/transports/meta.yaml b/resources/transports/meta.yaml index d73bb01..a691fae 100644 --- a/resources/transports/meta.yaml +++ b/resources/transports/meta.yaml @@ -1,7 +1,7 @@ id: transports input: transports: - schema: [{user: str, password: str, port: int!, key: str, name: str!}] + schema: [{user: str, password: str, port: int!, key: str, name: str!, trackers: [str]}] value: [] transports_id: schema: str! From f2466ce2331ac5943ecf195832916c57f7fcca2b Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 12 Oct 2015 10:44:48 +0200 Subject: [PATCH 4/9] run.yaml for torrent adjusted --- resources/transport_torrent/actions/run.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 resources/transport_torrent/actions/run.yaml diff --git a/resources/transport_torrent/actions/run.yaml b/resources/transport_torrent/actions/run.yaml new file mode 100644 index 0000000..76b0083 --- /dev/null +++ b/resources/transport_torrent/actions/run.yaml @@ -0,0 +1,9 @@ +- hosts: [{{ host }}] + sudo: yes + tasks: + - apt: + name: python-libtorrent + state: present + - copy: + src: {{scripts_dir}}/solar_torrent.py + dest: /var/tmp/solar_torrent.py From a503e835c7450b8539d60bcb1a716d18d1d83376 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 12 Oct 2015 11:17:16 +0200 Subject: [PATCH 5/9] Added torrent transport example (hosts_file) --- examples/torrent/example.py | 90 +++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 examples/torrent/example.py diff --git a/examples/torrent/example.py b/examples/torrent/example.py new file mode 100644 index 0000000..b08d99b --- /dev/null +++ b/examples/torrent/example.py @@ -0,0 +1,90 @@ +# To run: +# python example-riaks.py deploy +# python example-riaks.py add_haproxies +# solar changes stage +# solar changes process +# solar orch run-once last + + +import click +import sys +import time + +from solar.core import resource +from solar.core import signals +from solar.core import validation +from solar.core.resource import virtual_resource as vr +from solar import errors + +from solar.interfaces.db import get_db + +from solar.events.controls import React, Dep +from solar.events.api import add_event + + +db = get_db() + + + +def run(): + db.clear() + + node = vr.create('node', 'resources/ro_node', {'name': 'first' + str(time.time()), + 'ip': '10.0.0.3', + 'node_id': 'node1', + })[0] + + transports = vr.create('transports_node1', 'resources/transports')[0] + + ssh_transport = vr.create('ssh_transport', 'resources/transport_ssh', + {'ssh_key': '/vagrant/.vagrant/machines/solar-dev1/virtualbox/private_key', + 'ssh_user': 'vagrant'})[0] + + transports.connect(node, {}) + + + # it uses reverse mappings + ssh_transport.connect(transports, {'ssh_key': 'transports:key', + 'ssh_user': 'transports:user', + 'ssh_port': 'transports:port', + 'name': 'transports:name'}) + + hosts = vr.create('hosts_file', 'resources/hosts_file', {})[0] + + # let's add torrent transport for hosts file deployment (useless in real life) + + torrent_transport = vr.create('torrent_transport', + 'resources/transport_torrent', + {'trackers': ['http://tracker01-bud.infra.mirantis.net:8080']})[0] + + + transports_for_torrent = vr.create('transports_for_torrent', 'resources/transports')[0] + + transports_for_torrent.connect(torrent_transport, {}) + + ssh_transport.connect_with_events(transports_for_torrent, {'ssh_key': 'transports:key', + 'ssh_user': 'transports:user', + 'ssh_port': 'transports:port', + 'name': 'transports:name'}, + events={}) + + + transports_for_hosts = vr.create('transports_for_hosts', 'resources/transports')[0] + + torrent_transport.connect(transports_for_hosts, {'trackers': 'transports:trackers', + 'name': 'transports:name'}) + + ssh_transport.connect(transports_for_hosts, {'ssh_key': 'transports:key', + 'ssh_user': 'transports:user', + 'ssh_port': 'transports:port', + 'name': 'transports:name'}) + + transports_for_hosts.connect(hosts) + transports_for_hosts.connect_with_events(node, events={}) + + node.connect(hosts, { + 'ip': 'hosts:ip', + 'name': 'hosts:name' + }) + +run() From 3c72babcbcd039227073040af2abe01b47877bda Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Mon, 12 Oct 2015 13:50:38 +0200 Subject: [PATCH 6/9] Cleanup example --- examples/torrent/example.py | 51 +++++++++++++++---------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/examples/torrent/example.py b/examples/torrent/example.py index b08d99b..21dd102 100644 --- a/examples/torrent/example.py +++ b/examples/torrent/example.py @@ -6,59 +6,50 @@ # solar orch run-once last -import click -import sys import time -from solar.core import resource -from solar.core import signals -from solar.core import validation from solar.core.resource import virtual_resource as vr from solar import errors from solar.interfaces.db import get_db -from solar.events.controls import React, Dep -from solar.events.api import add_event - db = get_db() - def run(): db.clear() node = vr.create('node', 'resources/ro_node', {'name': 'first' + str(time.time()), - 'ip': '10.0.0.3', - 'node_id': 'node1', + 'ip': '10.0.0.3', + 'node_id': 'node1', })[0] transports = vr.create('transports_node1', 'resources/transports')[0] - ssh_transport = vr.create('ssh_transport', 'resources/transport_ssh', - {'ssh_key': '/vagrant/.vagrant/machines/solar-dev1/virtualbox/private_key', - 'ssh_user': 'vagrant'})[0] + ssh_transport = vr.create('ssh_transport', 'resources/transport_ssh', + {'ssh_key': '/vagrant/.vagrant/machines/solar-dev1/virtualbox/private_key', + 'ssh_user': 'vagrant'})[0] transports.connect(node, {}) - # it uses reverse mappings ssh_transport.connect(transports, {'ssh_key': 'transports:key', - 'ssh_user': 'transports:user', - 'ssh_port': 'transports:port', - 'name': 'transports:name'}) + 'ssh_user': 'transports:user', + 'ssh_port': 'transports:port', + 'name': 'transports:name'}) hosts = vr.create('hosts_file', 'resources/hosts_file', {})[0] - # let's add torrent transport for hosts file deployment (useless in real life) + # let's add torrent transport for hosts file deployment (useless in real + # life) - torrent_transport = vr.create('torrent_transport', - 'resources/transport_torrent', - {'trackers': ['http://tracker01-bud.infra.mirantis.net:8080']})[0] + torrent_transport = vr.create('torrent_transport', + 'resources/transport_torrent', + {'trackers': ['http://tracker01-bud.infra.mirantis.net:8080']})[0] - - transports_for_torrent = vr.create('transports_for_torrent', 'resources/transports')[0] + transports_for_torrent = vr.create( + 'transports_for_torrent', 'resources/transports')[0] transports_for_torrent.connect(torrent_transport, {}) @@ -68,16 +59,16 @@ def run(): 'name': 'transports:name'}, events={}) - - transports_for_hosts = vr.create('transports_for_hosts', 'resources/transports')[0] + transports_for_hosts = vr.create( + 'transports_for_hosts', 'resources/transports')[0] torrent_transport.connect(transports_for_hosts, {'trackers': 'transports:trackers', - 'name': 'transports:name'}) + 'name': 'transports:name'}) ssh_transport.connect(transports_for_hosts, {'ssh_key': 'transports:key', - 'ssh_user': 'transports:user', - 'ssh_port': 'transports:port', - 'name': 'transports:name'}) + 'ssh_user': 'transports:user', + 'ssh_port': 'transports:port', + 'name': 'transports:name'}) transports_for_hosts.connect(hosts) transports_for_hosts.connect_with_events(node, events={}) From fa010f3179d617c5e450a1b92f0ab0d0bf87e5e1 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 13 Oct 2015 13:22:00 +0200 Subject: [PATCH 7/9] Correct readme for torrent example --- examples/torrent/README.md | 21 +++++++++++++++++++++ examples/torrent/example.py | 8 -------- 2 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 examples/torrent/README.md diff --git a/examples/torrent/README.md b/examples/torrent/README.md new file mode 100644 index 0000000..5eadad5 --- /dev/null +++ b/examples/torrent/README.md @@ -0,0 +1,21 @@ +Example of using torrent transport with solar. Torrent is used to distribute task data. After fetching is finished torrent client forks and continues seeding. + + +The example contains single node with single host mapping + transports. + +Execute: +``` +python examples/torrent/example.py +solar changes stage +solar changes process +solar orch run-once last +``` + +Wait for finish: + +``` +solar orch report last -w 100 +``` + +After this you should see new entry in `/etc/hosts` file. + diff --git a/examples/torrent/example.py b/examples/torrent/example.py index 21dd102..6d8dabc 100644 --- a/examples/torrent/example.py +++ b/examples/torrent/example.py @@ -1,11 +1,3 @@ -# To run: -# python example-riaks.py deploy -# python example-riaks.py add_haproxies -# solar changes stage -# solar changes process -# solar orch run-once last - - import time from solar.core.resource import virtual_resource as vr From 48517f82e39eb74d47eac308837a5fd9b500c82f Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 13 Oct 2015 13:57:03 +0200 Subject: [PATCH 8/9] Added some info how torrent transport works --- examples/torrent/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/torrent/README.md b/examples/torrent/README.md index 5eadad5..13b6a33 100644 --- a/examples/torrent/README.md +++ b/examples/torrent/README.md @@ -19,3 +19,7 @@ solar orch report last -w 100 After this you should see new entry in `/etc/hosts` file. + +* All created torrents are in `/vagrant/torrents`, it doesn't need to be shared +* Initial seeding is done using torrent file +* Downloading and then seeding is always done with magnetlinks From 08e41b12289e9c17c0d28296c2b4a9aca7a57fea Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Tue, 13 Oct 2015 13:57:31 +0200 Subject: [PATCH 9/9] Changed torrent tracker to public --- examples/torrent/example.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/torrent/example.py b/examples/torrent/example.py index 6d8dabc..3719105 100644 --- a/examples/torrent/example.py +++ b/examples/torrent/example.py @@ -33,12 +33,13 @@ def run(): hosts = vr.create('hosts_file', 'resources/hosts_file', {})[0] - # let's add torrent transport for hosts file deployment (useless in real - # life) + # let's add torrent transport for hosts file deployment (useless in real life) torrent_transport = vr.create('torrent_transport', 'resources/transport_torrent', - {'trackers': ['http://tracker01-bud.infra.mirantis.net:8080']})[0] + {'trackers': ['udp://open.demonii.com:1337', + 'udp://tracker.openbittorrent.com:80']})[0] + # you could use any trackers as you want transports_for_torrent = vr.create( 'transports_for_torrent', 'resources/transports')[0]