From 5667ad5acb2ea0f899201812759ed7d78af7ce34 Mon Sep 17 00:00:00 2001 From: Sebastian Wuest Date: Wed, 5 Apr 2017 10:14:50 +0200 Subject: [PATCH 1/7] Init. for db migration. --- .../tomato/db/migrations/migration_0001.py | 218 ++++++++++++++++++ 1 file changed, 218 insertions(+) diff --git a/hostmanager/tomato/db/migrations/migration_0001.py b/hostmanager/tomato/db/migrations/migration_0001.py index 45680ad91..26ebb9e4d 100644 --- a/hostmanager/tomato/db/migrations/migration_0001.py +++ b/hostmanager/tomato/db/migrations/migration_0001.py @@ -1,3 +1,221 @@ +from django.db import models +from ...lib import db, attributes, logging +from ...lib.attributes import Attr +from ...lib.constants import ActionName, StateName, TechName + +kblang_options = {"en-us": "English (US)", + "en-gb": "English (GB)", + "de": "German", + "fr": "French", + "ja": "Japanese" +} + +def Connection(db.ChangesetMixin, attributes.Mixin, models.Model): + type = models.CharField(max_length=20, validators=[db.nameValidator], + choices=[(t, t) for t in ["bridge", "fixed_bridge"]]) # @ReservedAssignment + owner = models.ForeignKey(User, related_name='connections') + state = models.CharField(max_length=20, validators=[db.nameValidator]) + usageStatistics = models.OneToOneField(UsageStatistics, null=True, related_name='connection') + attrs = db.JSONField() + # elements: set of elements.Element + + +def Bridge(Connection): + bridge_attr = Attr("bridge", type="str") + bridge = bridge_attr.attribute() + + emulation_attr = Attr("emulation", desc="Enable emulation", type="bool", default=True) + emulation = emulation_attr.attribute() + + bandwidth_to_attr = Attr("bandwidth_to", desc="Bandwidth", unit="kbit/s", type="float", minValue=0, + maxValue=1000000, default=10000) + bandwidth_to = bandwidth_to_attr.attribute() + bandwidth_from_attr = Attr("bandwidth_from", desc="Bandwidth", unit="kbit/s", type="float", minValue=0, + maxValue=1000000, default=10000) + bandwidth_from = bandwidth_from_attr.attribute() + + lossratio_to_attr = Attr("lossratio_to", desc="Loss ratio", unit="%", type="float", minValue=0.0, maxValue=100.0, + default=0.0) + lossratio_to = lossratio_to_attr.attribute() + lossratio_from_attr = Attr("lossratio_from", desc="Loss ratio", unit="%", type="float", minValue=0.0, + maxValue=100.0, default=0.0) + lossratio_from = lossratio_from_attr.attribute() + + duplicate_to_attr = Attr("duplicate_to", desc="Duplication ratio", unit="%", type="float", minValue=0.0, + maxValue=100.0, default=0.0) + duplicate_to = duplicate_to_attr.attribute() + duplicate_from_attr = Attr("duplicate_from", desc="Duplication ratio", unit="%", type="float", minValue=0.0, + maxValue=100.0, default=0.0) + duplicate_from = duplicate_from_attr.attribute() + + corrupt_to_attr = Attr("corrupt_to", desc="Corruption ratio", unit="%", type="float", minValue=0.0, maxValue=100.0, + default=0.0) + corrupt_to = corrupt_to_attr.attribute() + corrupt_from_attr = Attr("corrupt_from", desc="Corruption ratio", unit="%", type="float", minValue=0.0, + maxValue=100.0, default=0.0) + corrupt_from = corrupt_from_attr.attribute() + + delay_to_attr = Attr("delay_to", desc="Delay", unit="ms", type="float", minValue=0.0, default=0.0) + delay_to = delay_to_attr.attribute() + delay_from_attr = Attr("delay_from", desc="Delay", unit="ms", type="float", minValue=0.0, default=0.0) + delay_from = delay_from_attr.attribute() + + jitter_to_attr = Attr("jitter_to", desc="Jitter", unit="ms", type="float", minValue=0.0, default=0.0) + jitter_to = jitter_to_attr.attribute() + jitter_from_attr = Attr("jitter_from", desc="Jitter", unit="ms", type="float", minValue=0.0, default=0.0) + jitter_from = jitter_from_attr.attribute() + + distribution_to_attr = Attr("distribution_to", desc="Distribution", type="str", + options={"uniform": "Uniform", "normal": "Normal", "pareto": "Pareto", + "paretonormal": "Pareto-Normal"}, default="uniform") + distribution_to = distribution_to_attr.attribute() + distribution_from_attr = Attr("distribution_from", desc="Distribution", type="str", + options={"uniform": "Uniform", "normal": "Normal", "pareto": "Pareto", + "paretonormal": "Pareto-Normal"}, default="uniform") + distribution_from = distribution_from_attr.attribute() + + capturing_attr = Attr("capturing", desc="Enable packet capturing", type="bool", default=False) + capturing = capturing_attr.attribute() + capture_filter_attr = Attr("capture_filter", desc="Packet filter expression", type="str", default="") + capture_filter = capture_filter_attr.attribute() + capture_port_attr = Attr("capture_port", type="int") + capture_port = capture_port_attr.attribute() + capture_mode_attr = Attr("capture_mode", desc="Capture mode", type="str", + options={"net": "Via network", "file": "For download"}, default="file") + capture_mode = capture_mode_attr.attribute() + capture_pid_attr = Attr("capture_pid", type="int") + capture_pid = capture_pid_attr.attribute() + + TYPE = "bridge" + +class Fixed_Bridge(Connection): + TYPE = "fixed_bridge" + +class Element(db.ChangesetMixin, attributes.Mixin, models.Model): + type = models.CharField(max_length=20, validators=[db.nameValidator], + choices=[(t, t) for t in TYPES.keys()]) # @ReservedAssignment + owner = models.ForeignKey(User, related_name='elements') + parent = models.ForeignKey('self', null=True, related_name='children') + connection = models.ForeignKey(Connection, null=True, related_name='elements') + usageStatistics = models.OneToOneField(UsageStatistics, null=True, related_name='element') + state = models.CharField(max_length=20, validators=[db.nameValidator]) + timeout = models.FloatField() + timeout_attr = Attr("timeout", desc="Timeout", states=[], type="float", null=False) + attrs = db.JSONField() + +class RexTFVElement: + rextfv_max_size = None + +class External_Network(Element): + network_attr = Attr("network", null=True) + network = models.ForeignKey(Network, null=True, related_name="instances") + +class KVM(RexTFVElement,Element): + vmid_attr = Attr("vmid", type="int") + vmid = vmid_attr.attribute() + websocket_port_attr = Attr("websocket_port", type="int") + websocket_port = websocket_port_attr.attribute() + websocket_pid_attr = Attr("websocket_pid", type="int") + websocket_pid = websocket_pid_attr.attribute() + vncport_attr = Attr("vncport", type="int") + vncport = vncport_attr.attribute() + vncpid_attr = Attr("vncpid", type="int") + vncpid = vncpid_attr.attribute() + vncpassword_attr = Attr("vncpassword", type="str") + vncpassword = vncpassword_attr.attribute() + cpus_attr = Attr("cpus", desc="Number of CPUs", states=[StateName.CREATED, StateName.PREPARED], type="int", minValue=1, maxValue=4, default=1) + cpus = cpus_attr.attribute() + ram_attr = Attr("ram", desc="RAM", unit="MB", states=[StateName.CREATED, StateName.PREPARED], type="int", minValue=64, maxValue=8192, default=256) + ram = ram_attr.attribute() + kblang_attr = Attr("kblang", desc="Keyboard language", states=[StateName.CREATED, StateName.PREPARED], type="str", options=kblang_options, default=None, null=True) + #["pt", "tr", "ja", "es", "no", "is", "fr-ca", "fr", "pt-br", "da", "fr-ch", "sl", "de-ch", "en-gb", "it", "en-us", "fr-be", "hu", "pl", "nl", "mk", "fi", "lt", "sv", "de"] + kblang = kblang_attr.attribute() + usbtablet_attr = Attr("usbtablet", desc="USB tablet mouse mode", states=[StateName.CREATED, StateName.PREPARED], type="bool", default=True) + usbtablet = usbtablet_attr.attribute() + template_attr = Attr("template", desc="Template", states=[StateName.CREATED, StateName.PREPARED], type="str", null=True) + template = models.ForeignKey(Template, null=True) + + +class KVMQM(RexTFVElement,Element): + vmid_attr = Attr("vmid", type="int") + vmid = vmid_attr.attribute() + websocket_port_attr = Attr("websocket_port", type="int") + websocket_port = websocket_port_attr.attribute() + websocket_pid_attr = Attr("websocket_pid", type="int") + websocket_pid = websocket_pid_attr.attribute() + vncport_attr = Attr("vncport", type="int") + vncport = vncport_attr.attribute() + vncpid_attr = Attr("vncpid", type="int") + vncpid = vncpid_attr.attribute() + vncpassword_attr = Attr("vncpassword", type="str") + vncpassword = vncpassword_attr.attribute() + cpus_attr = Attr("cpus", desc="Number of CPUs", states=[StateName.CREATED, StateName.PREPARED], type="int", minValue=1, maxValue=4, default=1) + cpus = cpus_attr.attribute() + ram_attr = Attr("ram", desc="RAM", unit="MB", states=[StateName.CREATED, StateName.PREPARED], type="int", minValue=64, maxValue=8192, default=256) + ram = ram_attr.attribute() + kblang_attr = Attr("kblang", desc="Keyboard language", states=[StateName.CREATED, StateName.PREPARED], type="str", options=kblang_options, default=None, null=True) + #["pt", "tr", "ja", "es", "no", "is", "fr-ca", "fr", "pt-br", "da", "fr-ch", "sl", "de-ch", "en-gb", "it", "en-us", "fr-be", "hu", "pl", "nl", "mk", "fi", "lt", "sv", "de"] + kblang = kblang_attr.attribute() + usbtablet_attr = Attr("usbtablet", desc="USB tablet mouse mode", states=[StateName.CREATED, StateName.PREPARED], type="bool", default=True) + usbtablet = usbtablet_attr.attribute() + template_attr = Attr("template", desc="Template", states=[StateName.CREATED, StateName.PREPARED], type="str", null=True) + template = models.ForeignKey(template.Template, null=True) + +class OpenVZ(RexTFVElement,Element): + vmid_attr = Attr("vmid", type="int") + vmid = vmid_attr.attribute() + websocket_port_attr = Attr("websocket_port", type="int") + websocket_port = websocket_port_attr.attribute() + websocket_pid_attr = Attr("websocket_pid", type="int") + websocket_pid = websocket_pid_attr.attribute() + vncport_attr = Attr("vncport", type="int") + vncport = vncport_attr.attribute() + vncpid_attr = Attr("vncpid", type="int") + vncpid = vncpid_attr.attribute() + vncpassword_attr = Attr("vncpassword", type="str") + vncpassword = vncpassword_attr.attribute() + ram_attr = Attr("ram", desc="RAM", unit="MB", type="int", minValue=64, maxValue=4096, default=256) + ram = ram_attr.attribute() + cpus_attr = Attr("cpus", desc="Number of CPUs", type="float", minValue=0.1, maxValue=4.0, default=1.0) + cpus = cpus_attr.attribute() + diskspace_attr = Attr("diskspace", desc="Disk space", unit="MB", type="int", minValue=512, maxValue=102400, default=10240) + diskspace = diskspace_attr.attribute() + rootpassword_attr = Attr("rootpassword", desc="Root password", type="str") + rootpassword = rootpassword_attr.attribute() + hostname_attr = Attr("hostname", desc="Hostname", type="str") + hostname = hostname_attr.attribute() + gateway4_attr = Attr("gateway4", desc="IPv4 gateway", type="str") + gateway4 = gateway4_attr.attribute() + gateway6_attr = Attr("gateway6", desc="IPv6 gateway", type="str") + gateway6 = gateway6_attr.attribute() + template_attr = Attr("template", desc="Template", states=[StateName.CREATED, StateName.PREPARED], type="str", null=True) + template = models.ForeignKey(Template, null=True) + +class Repy(elements.Element): + pid_attr = Attr("pid", type="int") + pid = pid_attr.attribute() + websocket_port_attr = Attr("websocket_port", type="int") + websocket_port = websocket_port_attr.attribute() + websocket_pid_attr = Attr("websocket_pid", type="int") + websocket_pid = websocket_pid_attr.attribute() + vncport_attr = Attr("vncport", type="int") + vncport = vncport_attr.attribute() + vncpid_attr = Attr("vncpid", type="int") + vncpid = vncpid_attr.attribute() + vncpassword_attr = Attr("vncpassword", type="str") + vncpassword = vncpassword_attr.attribute() + args_attr = Attr("args", desc="Arguments", states=[StateName.PREPARED], default=[]) + args = args_attr.attribute() + cpus_attr = Attr("cpus", desc="Number of CPUs", states=[StateName.PREPARED], type="float", minValue=0.01, maxValue=4.0, default=0.25) + cpus = cpus_attr.attribute() + ram_attr = Attr("ram", desc="RAM", unit="MB", states=[StateName.PREPARED], type="int", minValue=10, maxValue=4096, default=25) + ram = ram_attr.attribute() + bandwidth_attr = Attr("bandwidth", desc="Bandwidth", unit="bytes/s", states=[StateName.PREPARED], type="int", minValue=1024, maxValue=10000000000, default=1000000) + bandwidth = bandwidth_attr.attribute() + #TODO: use template ref instead of attr + template_attr = Attr("template", desc="Template", states=[StateName.PREPARED], type="str", null=True) + template = models.ForeignKey(Template, null=True) + def migrate(): pass \ No newline at end of file From 72da1d29a04751b69278ae9d5d40f1c6b499745a Mon Sep 17 00:00:00 2001 From: Sebastian Wuest Date: Sat, 22 Apr 2017 08:25:42 +0200 Subject: [PATCH 2/7] early version of the migration script --- .../tomato/db/migrations/migration_0001.py | 315 +++++++++++++----- 1 file changed, 227 insertions(+), 88 deletions(-) diff --git a/hostmanager/tomato/db/migrations/migration_0001.py b/hostmanager/tomato/db/migrations/migration_0001.py index 26ebb9e4d..0629ce8e5 100644 --- a/hostmanager/tomato/db/migrations/migration_0001.py +++ b/hostmanager/tomato/db/migrations/migration_0001.py @@ -10,91 +10,152 @@ "fr": "French", "ja": "Japanese" } +class User(models.Model): + name = models.CharField(max_length=255, unique=True) -def Connection(db.ChangesetMixin, attributes.Mixin, models.Model): - type = models.CharField(max_length=20, validators=[db.nameValidator], - choices=[(t, t) for t in ["bridge", "fixed_bridge"]]) # @ReservedAssignment - owner = models.ForeignKey(User, related_name='connections') - state = models.CharField(max_length=20, validators=[db.nameValidator]) - usageStatistics = models.OneToOneField(UsageStatistics, null=True, related_name='connection') - attrs = db.JSONField() - # elements: set of elements.Element - - -def Bridge(Connection): - bridge_attr = Attr("bridge", type="str") - bridge = bridge_attr.attribute() - - emulation_attr = Attr("emulation", desc="Enable emulation", type="bool", default=True) - emulation = emulation_attr.attribute() - - bandwidth_to_attr = Attr("bandwidth_to", desc="Bandwidth", unit="kbit/s", type="float", minValue=0, - maxValue=1000000, default=10000) - bandwidth_to = bandwidth_to_attr.attribute() - bandwidth_from_attr = Attr("bandwidth_from", desc="Bandwidth", unit="kbit/s", type="float", minValue=0, - maxValue=1000000, default=10000) - bandwidth_from = bandwidth_from_attr.attribute() - - lossratio_to_attr = Attr("lossratio_to", desc="Loss ratio", unit="%", type="float", minValue=0.0, maxValue=100.0, - default=0.0) - lossratio_to = lossratio_to_attr.attribute() - lossratio_from_attr = Attr("lossratio_from", desc="Loss ratio", unit="%", type="float", minValue=0.0, - maxValue=100.0, default=0.0) - lossratio_from = lossratio_from_attr.attribute() - - duplicate_to_attr = Attr("duplicate_to", desc="Duplication ratio", unit="%", type="float", minValue=0.0, - maxValue=100.0, default=0.0) - duplicate_to = duplicate_to_attr.attribute() - duplicate_from_attr = Attr("duplicate_from", desc="Duplication ratio", unit="%", type="float", minValue=0.0, - maxValue=100.0, default=0.0) - duplicate_from = duplicate_from_attr.attribute() - - corrupt_to_attr = Attr("corrupt_to", desc="Corruption ratio", unit="%", type="float", minValue=0.0, maxValue=100.0, - default=0.0) - corrupt_to = corrupt_to_attr.attribute() - corrupt_from_attr = Attr("corrupt_from", desc="Corruption ratio", unit="%", type="float", minValue=0.0, - maxValue=100.0, default=0.0) - corrupt_from = corrupt_from_attr.attribute() - - delay_to_attr = Attr("delay_to", desc="Delay", unit="ms", type="float", minValue=0.0, default=0.0) - delay_to = delay_to_attr.attribute() - delay_from_attr = Attr("delay_from", desc="Delay", unit="ms", type="float", minValue=0.0, default=0.0) - delay_from = delay_from_attr.attribute() - - jitter_to_attr = Attr("jitter_to", desc="Jitter", unit="ms", type="float", minValue=0.0, default=0.0) - jitter_to = jitter_to_attr.attribute() - jitter_from_attr = Attr("jitter_from", desc="Jitter", unit="ms", type="float", minValue=0.0, default=0.0) - jitter_from = jitter_from_attr.attribute() - - distribution_to_attr = Attr("distribution_to", desc="Distribution", type="str", - options={"uniform": "Uniform", "normal": "Normal", "pareto": "Pareto", - "paretonormal": "Pareto-Normal"}, default="uniform") - distribution_to = distribution_to_attr.attribute() - distribution_from_attr = Attr("distribution_from", desc="Distribution", type="str", - options={"uniform": "Uniform", "normal": "Normal", "pareto": "Pareto", - "paretonormal": "Pareto-Normal"}, default="uniform") - distribution_from = distribution_from_attr.attribute() - - capturing_attr = Attr("capturing", desc="Enable packet capturing", type="bool", default=False) - capturing = capturing_attr.attribute() - capture_filter_attr = Attr("capture_filter", desc="Packet filter expression", type="str", default="") - capture_filter = capture_filter_attr.attribute() - capture_port_attr = Attr("capture_port", type="int") - capture_port = capture_port_attr.attribute() - capture_mode_attr = Attr("capture_mode", desc="Capture mode", type="str", - options={"net": "Via network", "file": "For download"}, default="file") - capture_mode = capture_mode_attr.attribute() - capture_pid_attr = Attr("capture_pid", type="int") - capture_pid = capture_pid_attr.attribute() - - TYPE = "bridge" +class UsageStatistics(attributes.Mixin, models.Model): + begin = models.FloatField() #unix timestamp + #records: [UsageRecord] + attrs = db.JSONField() + +class UsageRecord(models.Model): + statistics = models.ForeignKey(UsageStatistics, related_name="records") + type = models.CharField(max_length=10, choices=[(t, t) for t in ["single", "5minutes", "hour", "day", "month", "year"]]) #@ReservedAssignment + begin = models.FloatField() #unix timestamp + end = models.FloatField() #unix timestamp + measurements = models.IntegerField() + #using fields to save space + memory = models.FloatField() #unit: bytes + diskspace = models.FloatField() #unit: bytes + traffic = models.FloatField() #unit: bytes + cputime = models.FloatField() #unit: cpu seconds + + +class Resource(db.ChangesetMixin, attributes.Mixin, models.Model): + type = models.CharField(max_length=20, validators=[db.nameValidator], choices=[(t, t) for t in ['template','network']]) #@ReservedAssignment + attrs = db.JSONField() + +class ResourceInstance(db.ChangesetMixin, attributes.Mixin, models.Model): + type = models.CharField(max_length=20, validators=[db.nameValidator], choices=[(t, t) for t in ['template','network']]) #@ReservedAssignment + num = models.IntegerField() + ownerElement = models.ForeignKey(Element, null=True) + ownerConnection = models.ForeignKey(Connection, null=True) + attrs = db.JSONField() + +class Network(Resource): + owner = models.ForeignKey(User, related_name='networks') + kind = models.CharField(max_length=50) + bridge = models.CharField(max_length=20) + preference = models.IntegerField(default=0) + +class Template(Resource): + owner = models.ForeignKey(User, related_name='templates') + tech = models.CharField(max_length=20) + name = models.CharField(max_length=50) + preference = models.IntegerField(default=0) + urls = attributes.attribute("urls", list) + checksum = attributes.attribute("checksum", str) + size = attributes.attribute("size", long) + popularity = attributes.attribute("popularity", float) + ready = attributes.attribute("ready", bool) + kblang = attributes.attribute("kblang", str, null=False, default="en-us") + + +class Connection(db.ChangesetMixin, attributes.Mixin, models.Model): + type = models.CharField(max_length=20, validators=[db.nameValidator], + choices=[(t, t) for t in ["bridge", "fixed_bridge"]]) # @ReservedAssignment + owner = models.ForeignKey(User, related_name='connections') + state = models.CharField(max_length=20, validators=[db.nameValidator]) + usageStatistics = models.OneToOneField(UsageStatistics, null=True, related_name='connection') + attrs = db.JSONField() + # elements: set of elements.Element + + +class Bridge(Connection): + bridge_attr = Attr("bridge", type="str") + bridge = bridge_attr.attribute() + + emulation_attr = Attr("emulation", desc="Enable emulation", type="bool", default=True) + emulation = emulation_attr.attribute() + + bandwidth_to_attr = Attr("bandwidth_to", desc="Bandwidth", unit="kbit/s", type="float", minValue=0, + maxValue=1000000, default=10000) + bandwidth_to = bandwidth_to_attr.attribute() + bandwidth_from_attr = Attr("bandwidth_from", desc="Bandwidth", unit="kbit/s", type="float", minValue=0, + maxValue=1000000, default=10000) + bandwidth_from = bandwidth_from_attr.attribute() + + lossratio_to_attr = Attr("lossratio_to", desc="Loss ratio", unit="%", type="float", minValue=0.0, maxValue=100.0, + default=0.0) + lossratio_to = lossratio_to_attr.attribute() + lossratio_from_attr = Attr("lossratio_from", desc="Loss ratio", unit="%", type="float", minValue=0.0, + maxValue=100.0, default=0.0) + lossratio_from = lossratio_from_attr.attribute() + + duplicate_to_attr = Attr("duplicate_to", desc="Duplication ratio", unit="%", type="float", minValue=0.0, + maxValue=100.0, default=0.0) + duplicate_to = duplicate_to_attr.attribute() + duplicate_from_attr = Attr("duplicate_from", desc="Duplication ratio", unit="%", type="float", minValue=0.0, + maxValue=100.0, default=0.0) + duplicate_from = duplicate_from_attr.attribute() + + corrupt_to_attr = Attr("corrupt_to", desc="Corruption ratio", unit="%", type="float", minValue=0.0, maxValue=100.0, + default=0.0) + corrupt_to = corrupt_to_attr.attribute() + corrupt_from_attr = Attr("corrupt_from", desc="Corruption ratio", unit="%", type="float", minValue=0.0, + maxValue=100.0, default=0.0) + corrupt_from = corrupt_from_attr.attribute() + + delay_to_attr = Attr("delay_to", desc="Delay", unit="ms", type="float", minValue=0.0, default=0.0) + delay_to = delay_to_attr.attribute() + delay_from_attr = Attr("delay_from", desc="Delay", unit="ms", type="float", minValue=0.0, default=0.0) + delay_from = delay_from_attr.attribute() + + jitter_to_attr = Attr("jitter_to", desc="Jitter", unit="ms", type="float", minValue=0.0, default=0.0) + jitter_to = jitter_to_attr.attribute() + jitter_from_attr = Attr("jitter_from", desc="Jitter", unit="ms", type="float", minValue=0.0, default=0.0) + jitter_from = jitter_from_attr.attribute() + + distribution_to_attr = Attr("distribution_to", desc="Distribution", type="str", + options={"uniform": "Uniform", "normal": "Normal", "pareto": "Pareto", + "paretonormal": "Pareto-Normal"}, default="uniform") + distribution_to = distribution_to_attr.attribute() + distribution_from_attr = Attr("distribution_from", desc="Distribution", type="str", + options={"uniform": "Uniform", "normal": "Normal", "pareto": "Pareto", + "paretonormal": "Pareto-Normal"}, default="uniform") + distribution_from = distribution_from_attr.attribute() + + capturing_attr = Attr("capturing", desc="Enable packet capturing", type="bool", default=False) + capturing = capturing_attr.attribute() + capture_filter_attr = Attr("capture_filter", desc="Packet filter expression", type="str", default="") + capture_filter = capture_filter_attr.attribute() + capture_port_attr = Attr("capture_port", type="int") + capture_port = capture_port_attr.attribute() + capture_mode_attr = Attr("capture_mode", desc="Capture mode", type="str", + options={"net": "Via network", "file": "For download"}, default="file") + capture_mode = capture_mode_attr.attribute() + capture_pid_attr = Attr("capture_pid", type="int") + capture_pid = capture_pid_attr.attribute() + + TYPE = "bridge" class Fixed_Bridge(Connection): - TYPE = "fixed_bridge" + TYPE = "fixed_bridge" + +ELEMENT_TYPES= ["kvm", "kvm_interface", + "kvmqm","kvmqm_interface", + "openvz","openvz_interface", + "lxc", "lxc_interface", + "repy","repy_interface", + "external_network", + "external_network_endpoint", + "tinc","tinc_vpn","tinc_endpoint", + "udp_endpoint","udp_tunnel", + "vpncloud","vpncloud_endpoint"] class Element(db.ChangesetMixin, attributes.Mixin, models.Model): type = models.CharField(max_length=20, validators=[db.nameValidator], - choices=[(t, t) for t in TYPES.keys()]) # @ReservedAssignment + choices=[(t, t) for t in ELEMENT_TYPES]) # @ReservedAssignment owner = models.ForeignKey(User, related_name='elements') parent = models.ForeignKey('self', null=True, related_name='children') connection = models.ForeignKey(Connection, null=True, related_name='elements') @@ -102,14 +163,14 @@ class Element(db.ChangesetMixin, attributes.Mixin, models.Model): state = models.CharField(max_length=20, validators=[db.nameValidator]) timeout = models.FloatField() timeout_attr = Attr("timeout", desc="Timeout", states=[], type="float", null=False) - attrs = db.JSONField() + attrs = db.JSONField() class RexTFVElement: - rextfv_max_size = None + rextfv_max_size = None class External_Network(Element): network_attr = Attr("network", null=True) - network = models.ForeignKey(Network, null=True, related_name="instances") + network = models.ForeignKey(Network, null=True, related_name="instances") class KVM(RexTFVElement,Element): vmid_attr = Attr("vmid", type="int") @@ -134,8 +195,18 @@ class KVM(RexTFVElement,Element): usbtablet_attr = Attr("usbtablet", desc="USB tablet mouse mode", states=[StateName.CREATED, StateName.PREPARED], type="bool", default=True) usbtablet = usbtablet_attr.attribute() template_attr = Attr("template", desc="Template", states=[StateName.CREATED, StateName.PREPARED], type="str", null=True) - template = models.ForeignKey(Template, null=True) + template = models.ForeignKey(Template, null=True) +class KVM_Interface(Element): + num_attr = Attr("num", type="int") + num = num_attr.attribute() + name_attr = Attr("name", desc="Name", type="str", regExp="^eth[0-9]+$", states=[StateName.CREATED]) + mac_attr = Attr("mac", desc="MAC Address", type="str") + mac = mac_attr.attribute() + ipspy_pid_attr = Attr("ipspy_pid", type="int") + ipspy_pid = ipspy_pid_attr.attribute() + used_addresses_attr = Attr("used_addresses", type=list, default=[]) + used_addresses = used_addresses_attr.attribute() class KVMQM(RexTFVElement,Element): vmid_attr = Attr("vmid", type="int") @@ -160,7 +231,18 @@ class KVMQM(RexTFVElement,Element): usbtablet_attr = Attr("usbtablet", desc="USB tablet mouse mode", states=[StateName.CREATED, StateName.PREPARED], type="bool", default=True) usbtablet = usbtablet_attr.attribute() template_attr = Attr("template", desc="Template", states=[StateName.CREATED, StateName.PREPARED], type="str", null=True) - template = models.ForeignKey(template.Template, null=True) + template = models.ForeignKey(Template, null=True) + +class KVMQM_Interface(Element): + num_attr = Attr("num", type="int") + num = num_attr.attribute() + name_attr = Attr("name", desc="Name", type="str", regExp="^eth[0-9]+$", states=[StateName.CREATED]) + mac_attr = Attr("mac", desc="MAC Address", type="str") + mac = mac_attr.attribute() + ipspy_pid_attr = Attr("ipspy_pid", type="int") + ipspy_pid = ipspy_pid_attr.attribute() + used_addresses_attr = Attr("used_addresses", type=list, default=[]) + used_addresses = used_addresses_attr.attribute() class OpenVZ(RexTFVElement,Element): vmid_attr = Attr("vmid", type="int") @@ -190,9 +272,25 @@ class OpenVZ(RexTFVElement,Element): gateway6_attr = Attr("gateway6", desc="IPv6 gateway", type="str") gateway6 = gateway6_attr.attribute() template_attr = Attr("template", desc="Template", states=[StateName.CREATED, StateName.PREPARED], type="str", null=True) - template = models.ForeignKey(Template, null=True) + template = models.ForeignKey(Template, null=True) + +class OpenVZ_Interface(Element): + name_attr = Attr("name", desc="Name", type="str", regExp="^eth[0-9]+$", states=[StateName.CREATED]) + name = name_attr.attribute() + ip4address_attr = Attr("ip4address", desc="IPv4 address", type="str") + ip4address = ip4address_attr.attribute() + ip6address_attr = Attr("ip6address", desc="IPv6 address", type="str") + ip6address = ip6address_attr.attribute() + use_dhcp_attr = Attr("use_dhcp", desc="Use DHCP", type="bool", default=False) + use_dhcp = use_dhcp_attr.attribute() + mac_attr = Attr("mac", desc="MAC Address", type="str") + mac = mac_attr.attribute() + ipspy_pid_attr = Attr("ipspy_pid", type="int") + ipspy_pid = ipspy_pid_attr.attribute() + used_addresses_attr = Attr("used_addresses", type=list, default=[]) + used_addresses = used_addresses_attr.attribute() -class Repy(elements.Element): +class Repy(Element): pid_attr = Attr("pid", type="int") pid = pid_attr.attribute() websocket_port_attr = Attr("websocket_port", type="int") @@ -215,7 +313,48 @@ class Repy(elements.Element): bandwidth = bandwidth_attr.attribute() #TODO: use template ref instead of attr template_attr = Attr("template", desc="Template", states=[StateName.PREPARED], type="str", null=True) - template = models.ForeignKey(Template, null=True) + template = models.ForeignKey(Template, null=True) + +class Repy_Interface(Element): + name_attr = Attr("name", desc="Name", type="str", regExp="^eth[0-9]+$") + name = name_attr.attribute() + ipspy_pid_attr = Attr("ipspy_pid", type="int") + ipspy_pid = ipspy_pid_attr.attribute() + used_addresses_attr = Attr("used_addresses", type=list, default=[]) + used_addresses = used_addresses_attr.attribute() + +class Tinc(Element): + port_attr = Attr("port", type="int") + port = port_attr.attribute() + path_attr = Attr("path") + path = path_attr.attribute() + mode_attr = Attr("mode", desc="Mode", states=[StateName.CREATED], options={"hub": "Hub (broadcast)", "switch": "Switch (learning)"}, default="switch") + mode = mode_attr.attribute() + privkey_attr = Attr("privkey", desc="Private key") + privkey = privkey_attr.attribute() + pubkey_attr = Attr("pubkey", desc="Public key") + pubkey = pubkey_attr.attribute() + peers_attr = Attr("peers", desc="Peers", states=[StateName.CREATED], default=[]) + peers = peers_attr.attribute() + +class UDP_Tunnel(Element): + pid_attr = Attr("pid", type="int") + pid = pid_attr.attribute() + port_attr = Attr("port", type="int") + port = port_attr.attribute() + connect_attr = Attr("connect", desc="Connect to", states=[StateName.CREATED], type="str", null=True, default=None) + connect = connect_attr.attribute() + +class VpnCloud(Element): + port_attr = Attr("port", type="int") + port = port_attr.attribute() + pid_attr = Attr("pid", type="int", null=True) + pid = pid_attr.attribute() + network_id_attr = Attr("network_id", type="int") + network_id = network_id_attr.attribute() + peers_attr = Attr("peers", desc="Peers", states=[StateName.CREATED], default=[]) + peers = peers_attr.attribute() def migrate(): - pass \ No newline at end of file + + pass \ No newline at end of file From a298c81f2bfdcd967085f2d8f356e3e5c0c5af68 Mon Sep 17 00:00:00 2001 From: Sebastian Wuest Date: Tue, 2 May 2017 12:08:15 +0200 Subject: [PATCH 3/7] Work in progress --- hostmanager/tomato/db/migrations/migration_0001.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hostmanager/tomato/db/migrations/migration_0001.py b/hostmanager/tomato/db/migrations/migration_0001.py index 0629ce8e5..dea9f69e9 100644 --- a/hostmanager/tomato/db/migrations/migration_0001.py +++ b/hostmanager/tomato/db/migrations/migration_0001.py @@ -356,5 +356,16 @@ class VpnCloud(Element): peers = peers_attr.attribute() def migrate(): + from ...elements import Element as El + for element in Element.objects.filter(**kwargs) + element = element.upcast() + + + new_element = El.create({ + "type": element.type, + "owner": + "parent" + }) + pass \ No newline at end of file From 09e7eb7425ae5d13a89d71a6cb7208c5971bf565 Mon Sep 17 00:00:00 2001 From: Sebastian Wuest Date: Fri, 26 May 2017 14:04:01 +0200 Subject: [PATCH 4/7] Work in progress, added a lot of class migrations but without correct relationships. --- .../tomato/db/migrations/migration_0001.py | 265 +++++++++++++++++- 1 file changed, 257 insertions(+), 8 deletions(-) diff --git a/hostmanager/tomato/db/migrations/migration_0001.py b/hostmanager/tomato/db/migrations/migration_0001.py index dea9f69e9..ada463589 100644 --- a/hostmanager/tomato/db/migrations/migration_0001.py +++ b/hostmanager/tomato/db/migrations/migration_0001.py @@ -361,11 +361,260 @@ def migrate(): element = element.upcast() - new_element = El.create({ - "type": element.type, - "owner": - "parent" - }) - - - pass \ No newline at end of file + userMapping = {} + from ...user import User as Us + for user in User.objects.filter(): + newUser = Us(name=user.name) + newUser.save() + userMapping[user]=newUser + + resourceMapping = {} + from ...resources import Resource as Re + for resource in Resource.objects.filter(): + if resource.type not in ["network","template"]: + newResource = Re(attrs=resource.attrs,type=resource.type) + newResource.save() + resourceMapping[resource] = newResource + + templateMapping = {} + from ...resources.template import Template as Te + for template in Template.objects.filter(): + newTemplate = Te(attrs=resource.attrs, + type=resource.type, + owner=userMapping[template.owner], + ownerId=userMapping[template.owner].id, + tech=template.tech, + name=template.name, + preference=template.preference, + urls=template.urls, + checksum=template.checksum, + size=template.size, + popularity=template.popularity, + ready=template.ready, + kblang=template.kblang) + newTemplate.save() + templateMapping[template] = newTemplate + + networkMapping ={} + from ...resources.network import Network as Ne + for network in Network.objects.filter(): + newNetwork = Ne(attrs=resource.attrs, + type=resource.type, + owner=userMapping[network.owner], + ownerId=userMapping[network.owner].id, + kind=network.kind, + bridge=network.bridge, + preference=network.preference) + networkMapping[network] = newNetwork + + externalNetworkMapping = {} + from ...elements.external_network import External_Network as ExNe + for externalNetwork in External_Network.objects.filter(): + newExternalNetwork = ExNe( + state=externalNetwork.state, + timeout=externalNetwork.timeout, + network=networkMapping[externalNetwork.network], + networkId=networkMapping[externalNetwork.network].id + ) + externalNetworkMapping[externalNetwork] = newExternalNetwork + + + kvmqmMapping = {} + from ...elements.kvmqm import KVMQM as New_KVMQM + for kvmqm in KVMQM.objects.filter(): + newKVMQM = New_KVMQM( + state=kvmqm.state, + timeout=kvmqm.timeout, + vmid = kvmqm.vmid, + websocket_port = kvmqm.websocket_port, + websocket_pid = kvmqm.websocket_pid, + vncport = kvmqm.vncport, + vncpid = kvmqm.vncpid, + vncpassword = kvmqm.vncpassword, + cpus = kvmqm.cpus, + ram = kvmqm.ram, + kblang = kvmqm.kblang, + usbtablet = kvmqm.usbtablet, + template = kvmqm.template, + templateId = kvmqm.templateId + ) + newKVMQM.save() + kvmqmMapping[kvmqm] = newKVMQM + + kvmqm_interfaceMapping = {} + from ...elements.kvmqm import KVMQM_Interface as KVQM_Interface + for kvmqm_interface in KVMQM_Interface.objects.filter(): + newKVMQMInterface = KVQM_Interface( + state=kvmqm_interface.state, + timeout=kvmqm_interface.timeout, + num=kvmqm_interface.num, + name=kvmqm_interface.name, + mac=kvmqm_interface.mac, + ipspy_pid=kvmqm_interface.ipspy_id, + used_addresses=kvmqm_interface.used_addresses + ) + newKVMQMInterface.save() + kvmqm_interfaceMapping[kvmqm_interface] = newKVMQMInterface + + kvmMapping = {} + from ...elements.kvm import KVM as New_KVM + for kvm in KVM.objects.filter(): + newKVM = New_KVM( + state=kvm.state, + timeout=kvm.timeout, + vmid=kvm.vmid, + websocket_port=kvm.websocket_port, + websocket_pid=kvm.websocket_pid, + vncport=kvm.vncport, + vncpid=kvm.vncpid, + vncpassword=kvm.vncpassword, + cpus=kvm.cpus, + ram=kvm.ram, + kblang=kvm.kblang, + usbtablet=kvm.usbtablet, + template=kvm.template, + templateId=kvm.templateId + ) + newKVM.save() + kvmMapping[kvm] = newKVM + + kvm_interfaceMapping = {} + from ...elements.kvm import KVM_Interface as KV_Interface + for kvm_interface in KVM.objects.filter(): + newKVMInterface = KV_Interface( + state=kvm_interface.state, + timeout=kvm_interface.timeout, + num=kvm_interface.num, + name=kvm_interface.name, + mac=kvm_interface.mac, + ipspy_pid=kvm_interface.ipspy_id, + used_addresses=kvm_interface.used_addresses + ) + newKVMInterface.save() + kvm_interfaceMapping[kvm_interface] = newKVMInterface + + openvzMapping = {} + from ...elements.openvz import OpenVZ as New_OpenVZ + for openvz in OpenVZ.objects.filter(): + newOpenVZ = New_OpenVZ( + state=openvz.state, + timeout=openvz.timeout, + vmid=openvz.vmid, + websocket_port = openvz.websocket_port, + websocket_pid = openvz.websocket_pid, + vncport = openvz.vncport, + vncpid = openvz.vncpid, + vncpassword = openvz.vncpassword, + cpus = openvz.cpus, + ram = openvz.ram, + diskspace = openvz.diskspace, + rootpassword = openvz.rootpassword, + gateway4 = openvz.gateway4, + hostname = openvz.hostname, + gateway6 = openvz.gateway6, + usbtablet = openvz.usbtablet, + ) + newOpenVZ.save() + openvzMapping[openvz] = newOpenVZ + + openvz_interfaceMapping={} + from ...elements.openvz import OpenVZ_Interface as New_OpenVZ_Interface + for openvz_interface in OpenVZ_Interface.objects.filter(): + newOpenVz_Interface = New_OpenVZ_Interface( + state=openvz_interface.state, + timeout=openvz_interface.timeout, + name=openvz_interface.name, + ip4address=openvz_interface.ip4address, + ip6address=openvz_interface.ip6address, + use_dhcp=openvz_interface.use_dhcp, + mac=openvz_interface.mac, + ipspy_pid=openvz_interface.ipspy_pid, + used_addresses=openvz_interface.used_addresses + ) + newOpenVz_Interface.save() + openvz_interfaceMapping[openvz_interface]=newOpenVz_Interface + + repyMapping={} + from ...elements.repy import Repy as RP + for repy in Repy.objects.filter(): + newRepy = RP( + state=repy.state, + timeout=repy.timeout, + pid=repy.pid, + websocket_port=repy.websocket_port, + websocket_pid=repy.websocket_pid, + vncport=repy.vncport, + vncpid=repy.vncpid, + vncpassword=repy.vncpassword, + args=repy.args, + args_doc=repy.args_doc, + cpus=repy.cpus, + ram=repy.ram, + bandwidth=repy.bandwidth + ) + newRepy.save() + repyMapping[repy] = newRepy + + repy_interfaceMapping={} + from ...elements.repy import Repy_Interface as New_Repy_Interface + for repy_interface in Repy_Interface.objects.filter(): + newRepyInterface = New_Repy_Interface( + state=repy_interface.state, + timeout=repy_interface.timeout, + name=repy_interface.name, + ipspy_pid=repy_interface.ipspy_pid, + used_addresses=repy_interface.used_addresses, + ) + newRepyInterface.save() + repy_interfaceMapping[repy_interface] = newRepyInterface + + + tincMapping={} + from ...elements.tinc import Tinc as NewTc + for tinc in Tinc.objects.filter(): + newTinc = NewTc( + state=tinc.state, + timeout=tinc.timeout, + port=tinc.port, + path=tinc.path, + mode=tinc.mode, + privkey=tinc.privkey, + pubkey=tinc.pubkey, + peers=tinc.peers + ) + newTinc.save() + tincMapping[tinc] = newTinc + + + udpTunnelMapping={} + from ...elements.udp_tunnel import UDP_Tunnel as NewUDP_Tunnel + for udp_tunnel in UDP_Tunnel.objects.filter(): + newUDP_Tunnel = NewUDP_Tunnel( + state=udp_tunnel.state, + timeout=udp_tunnel.timeout, + pid=udp_tunnel.pid, + port=udp_tunnel.port, + connect=udp_tunnel.connect + ) + newUDP_Tunnel.save() + udpTunnelMapping[udp_tunnel] = newUDP_Tunnel + + + vpnCloudMapping={} + from ...elements.vpncloud import VpnCloud as NewVPNCloud + for vpncloud in VpnCloud.objects.filter(): + newVPNcloud = NewVPNCloud( + state=vpncloud.state, + timeout=vpncloud.timeout, + port=vpncloud.port, + pid=vpncloud.pid, + network_id=vpncloud.network_id, + peers=vpncloud.peers + ) + newVPNcloud.save() + vpnCloudMapping[vpncloud] = newVPNcloud + + + #TODO: Add connection(s) migration + #Test everythin once + #Add correct referencing of new objects for mongodb \ No newline at end of file From 1d1be2a83cf0499660a88287453b9ad8247865f7 Mon Sep 17 00:00:00 2001 From: Sebastian Wuest Date: Fri, 2 Jun 2017 10:20:20 +0200 Subject: [PATCH 5/7] Untested first full version of the migration script. --- .../tomato/db/migrations/migration_0001.py | 350 +++++++++++++++--- 1 file changed, 299 insertions(+), 51 deletions(-) diff --git a/hostmanager/tomato/db/migrations/migration_0001.py b/hostmanager/tomato/db/migrations/migration_0001.py index ada463589..d39842fa2 100644 --- a/hostmanager/tomato/db/migrations/migration_0001.py +++ b/hostmanager/tomato/db/migrations/migration_0001.py @@ -355,31 +355,35 @@ class VpnCloud(Element): peers_attr = Attr("peers", desc="Peers", states=[StateName.CREATED], default=[]) peers = peers_attr.attribute() +''' +Migration through 2 Steps: +1. Create old elements in the new database with all attributes except for references +2. Update all elements to integrate the old references and relationships between them. +''' + def migrate(): - from ...elements import Element as El - for element in Element.objects.filter(**kwargs) - element = element.upcast() + #Create old elements in the new database with all attributes except for references userMapping = {} - from ...user import User as Us + from ...user import User as NewUser for user in User.objects.filter(): - newUser = Us(name=user.name) + newUser = NewUser(name=user.name) newUser.save() userMapping[user]=newUser resourceMapping = {} - from ...resources import Resource as Re + from ...resources import Resource as NewResource for resource in Resource.objects.filter(): if resource.type not in ["network","template"]: - newResource = Re(attrs=resource.attrs,type=resource.type) + newResource = NewResource(attrs=resource.attrs, + type=resource.type) newResource.save() resourceMapping[resource] = newResource - templateMapping = {} - from ...resources.template import Template as Te + from ...resources.template import Template as NewTemplate for template in Template.objects.filter(): - newTemplate = Te(attrs=resource.attrs, + newTemplate = NewTemplate(attrs=resource.attrs, type=resource.type, owner=userMapping[template.owner], ownerId=userMapping[template.owner].id, @@ -393,36 +397,38 @@ def migrate(): ready=template.ready, kblang=template.kblang) newTemplate.save() - templateMapping[template] = newTemplate + resourceMapping[template] = newTemplate - networkMapping ={} - from ...resources.network import Network as Ne + from ...resources.network import Network as NewNetwork for network in Network.objects.filter(): - newNetwork = Ne(attrs=resource.attrs, + newNetwork = NewNetwork(attrs=resource.attrs, type=resource.type, owner=userMapping[network.owner], ownerId=userMapping[network.owner].id, kind=network.kind, bridge=network.bridge, preference=network.preference) - networkMapping[network] = newNetwork + resourceMapping[network] = newNetwork - externalNetworkMapping = {} - from ...elements.external_network import External_Network as ExNe + elementMapping = {} + from ...elements.external_network import External_Network as NewExternalNetwork for externalNetwork in External_Network.objects.filter(): - newExternalNetwork = ExNe( + newExternalNetwork = NewExternalNetwork( + owner=userMapping[externalNetwork.owner], + ownerId = userMapping[externalNetwork.owner].id, state=externalNetwork.state, timeout=externalNetwork.timeout, - network=networkMapping[externalNetwork.network], - networkId=networkMapping[externalNetwork.network].id + network=resourceMapping[externalNetwork.network], + networkId=resourceMapping[externalNetwork.network].id ) - externalNetworkMapping[externalNetwork] = newExternalNetwork + elementMapping[externalNetwork] = newExternalNetwork - kvmqmMapping = {} from ...elements.kvmqm import KVMQM as New_KVMQM for kvmqm in KVMQM.objects.filter(): newKVMQM = New_KVMQM( + owner=userMapping[kvmqm.owner], + ownerId = userMapping[kvmqm.owner].id, state=kvmqm.state, timeout=kvmqm.timeout, vmid = kvmqm.vmid, @@ -435,16 +441,17 @@ def migrate(): ram = kvmqm.ram, kblang = kvmqm.kblang, usbtablet = kvmqm.usbtablet, - template = kvmqm.template, - templateId = kvmqm.templateId + template = resourceMapping[kvmqm.template], + templateId = resourceMapping[kvmqm.template].id ) newKVMQM.save() - kvmqmMapping[kvmqm] = newKVMQM + elementMapping[kvmqm] = newKVMQM - kvmqm_interfaceMapping = {} from ...elements.kvmqm import KVMQM_Interface as KVQM_Interface for kvmqm_interface in KVMQM_Interface.objects.filter(): newKVMQMInterface = KVQM_Interface( + owner=userMapping[kvmqm_interface.owner], + ownerId=userMapping[kvmqm_interface.owner].id, state=kvmqm_interface.state, timeout=kvmqm_interface.timeout, num=kvmqm_interface.num, @@ -454,12 +461,13 @@ def migrate(): used_addresses=kvmqm_interface.used_addresses ) newKVMQMInterface.save() - kvmqm_interfaceMapping[kvmqm_interface] = newKVMQMInterface + elementMapping[kvmqm_interface] = newKVMQMInterface - kvmMapping = {} from ...elements.kvm import KVM as New_KVM for kvm in KVM.objects.filter(): newKVM = New_KVM( + owner=userMapping[kvm.owner], + ownerId = userMapping[kvm.owner].id, state=kvm.state, timeout=kvm.timeout, vmid=kvm.vmid, @@ -472,16 +480,17 @@ def migrate(): ram=kvm.ram, kblang=kvm.kblang, usbtablet=kvm.usbtablet, - template=kvm.template, - templateId=kvm.templateId + template = resourceMapping[kvm.template], + templateId = resourceMapping[kvm.template].id ) newKVM.save() - kvmMapping[kvm] = newKVM + elementMapping[kvm] = newKVM - kvm_interfaceMapping = {} from ...elements.kvm import KVM_Interface as KV_Interface for kvm_interface in KVM.objects.filter(): newKVMInterface = KV_Interface( + owner=userMapping[kvm_interface.owner], + ownerId=userMapping[kvm_interface.owner].id, state=kvm_interface.state, timeout=kvm_interface.timeout, num=kvm_interface.num, @@ -491,12 +500,13 @@ def migrate(): used_addresses=kvm_interface.used_addresses ) newKVMInterface.save() - kvm_interfaceMapping[kvm_interface] = newKVMInterface + elementMapping[kvm_interface] = newKVMInterface - openvzMapping = {} from ...elements.openvz import OpenVZ as New_OpenVZ for openvz in OpenVZ.objects.filter(): newOpenVZ = New_OpenVZ( + owner=userMapping[openvz.owner], + ownerId=userMapping[openvz.owner].id, state=openvz.state, timeout=openvz.timeout, vmid=openvz.vmid, @@ -512,15 +522,17 @@ def migrate(): gateway4 = openvz.gateway4, hostname = openvz.hostname, gateway6 = openvz.gateway6, - usbtablet = openvz.usbtablet, + template = resourceMapping[openvz.template], + templateId = resourceMapping[openvz.template].id ) newOpenVZ.save() - openvzMapping[openvz] = newOpenVZ + elementMapping[openvz] = newOpenVZ - openvz_interfaceMapping={} from ...elements.openvz import OpenVZ_Interface as New_OpenVZ_Interface for openvz_interface in OpenVZ_Interface.objects.filter(): newOpenVz_Interface = New_OpenVZ_Interface( + owner=userMapping[openvz_interface.owner], + ownerId=userMapping[openvz_interface.owner].id, state=openvz_interface.state, timeout=openvz_interface.timeout, name=openvz_interface.name, @@ -532,12 +544,13 @@ def migrate(): used_addresses=openvz_interface.used_addresses ) newOpenVz_Interface.save() - openvz_interfaceMapping[openvz_interface]=newOpenVz_Interface + elementMapping[openvz_interface]=newOpenVz_Interface - repyMapping={} from ...elements.repy import Repy as RP for repy in Repy.objects.filter(): newRepy = RP( + owner=userMapping[repy.owner], + ownerId=userMapping[repy.owner].id, state=repy.state, timeout=repy.timeout, pid=repy.pid, @@ -547,18 +560,20 @@ def migrate(): vncpid=repy.vncpid, vncpassword=repy.vncpassword, args=repy.args, - args_doc=repy.args_doc, cpus=repy.cpus, ram=repy.ram, - bandwidth=repy.bandwidth + bandwidth=repy.bandwidth, + template = resourceMapping[repy.template], + templateId = resourceMapping[repy.template].id ) newRepy.save() - repyMapping[repy] = newRepy + elementMapping[repy] = newRepy - repy_interfaceMapping={} from ...elements.repy import Repy_Interface as New_Repy_Interface for repy_interface in Repy_Interface.objects.filter(): newRepyInterface = New_Repy_Interface( + owner=userMapping[repy_interface.owner], + ownerId = userMapping[repy_interface.owner].id, state=repy_interface.state, timeout=repy_interface.timeout, name=repy_interface.name, @@ -566,13 +581,14 @@ def migrate(): used_addresses=repy_interface.used_addresses, ) newRepyInterface.save() - repy_interfaceMapping[repy_interface] = newRepyInterface + elementMapping[repy_interface] = newRepyInterface - tincMapping={} from ...elements.tinc import Tinc as NewTc for tinc in Tinc.objects.filter(): newTinc = NewTc( + owner=userMapping[tinc.owner], + ownerId = userMapping[tinc.owner].id, state=tinc.state, timeout=tinc.timeout, port=tinc.port, @@ -583,13 +599,14 @@ def migrate(): peers=tinc.peers ) newTinc.save() - tincMapping[tinc] = newTinc + elementMapping[tinc] = newTinc - udpTunnelMapping={} from ...elements.udp_tunnel import UDP_Tunnel as NewUDP_Tunnel for udp_tunnel in UDP_Tunnel.objects.filter(): newUDP_Tunnel = NewUDP_Tunnel( + owner=userMapping[udp_tunnel.owner], + ownerId=userMapping[udp_tunnel.owner].id, state=udp_tunnel.state, timeout=udp_tunnel.timeout, pid=udp_tunnel.pid, @@ -597,13 +614,14 @@ def migrate(): connect=udp_tunnel.connect ) newUDP_Tunnel.save() - udpTunnelMapping[udp_tunnel] = newUDP_Tunnel + elementMapping[udp_tunnel] = newUDP_Tunnel - vpnCloudMapping={} from ...elements.vpncloud import VpnCloud as NewVPNCloud for vpncloud in VpnCloud.objects.filter(): newVPNcloud = NewVPNCloud( + owner=userMapping[vpncloud.owner], + ownerId=userMapping[vpncloud.owner].id, state=vpncloud.state, timeout=vpncloud.timeout, port=vpncloud.port, @@ -612,9 +630,239 @@ def migrate(): peers=vpncloud.peers ) newVPNcloud.save() - vpnCloudMapping[vpncloud] = newVPNcloud + elementMapping[vpncloud] = newVPNcloud + + + connectionMapping={} + from ...connections.bridge import Bridge as NewBridge + for bridge in Bridge.objects.filte(): + newBridge = NewBridge( + owner=userMapping[bridge.owner], + ownerId = userMapping[bridge.owner].id, + state = bridge.state, + bridge= bridge.bridge, + emulation = bridge.emulation, + bandwidth_to = bridge.bandwidth_to, + bandwidth_from = bridge.bandwidth_from, + lossratio_to = bridge.lossratio_to, + lossratio_from = bridge.lossratio_from, + duplicate_to = bridge.duplicate_to, + duplicate_from = bridge.duplicate_from, + corrupt_to = bridge.corrupt_to, + corrupt_from = bridge.corrupt_from, + delay_to = bridge.delay_to, + delay_from = bridge.delay_from, + jitter_to = bridge.jitter_to, + jitter_from = bridge.jitter_from, + distribution_to = bridge.distribution_to, + distribution_from = bridge.distribution_from, + capturing = bridge.capturing, + capture_filter = bridge.capture_filter, + capture_port = bridge.capture_port, + capture_mode = bridge.capture_mode, + capture_pid = bridge.capture_pid) + newBridge.save() + connectionMapping[bridge] = newBridge + + from ...connections.fixed_bridge import Fixed_Bridge as NewFixedBridge + for fixedBridge in Fixed_Bridge.objects.filter(): + newFixedBridge = NewFixedBridge( + owner=userMapping[fixedBridge.owner], + ownerId = userMapping[fixedBridge.owner].id, + state = fixedBridge.state) + newFixedBridge.save() + connectionMapping[fixedBridge]=newFixedBridge + + usageStatisticsMapping={} + from ...accounting import UsageStatistics as NewUsageStatistics + for usageStatistic in UsageStatistics.objects.filter(): + newUsageStatistic = NewUsageStatistics( + begin =usageStatistic.begin, + attrs=usageStatistic.attrs + ) + newUsageStatistic.save() + usageStatisticsMapping[usageStatistic] = newUsageStatistic + + usageRecordMapping={} + from ...accounting import UsageRecord as NewUsageRecord + for usageRecord in UsageRecord.objects.filter(): + newUsageRecord = NewUsageRecord( + statistics=usageStatisticsMapping[usageRecord.statistics], + type=usageRecord.type, + begin=usageRecord.begin, + end=usageRecord.end, + measurements=usageRecord.measurements, + memory=usageRecord.memory, + diskspace=usageRecord.diskspace, + traffic=usageRecord.traffic, + cputime=usageRecord.cputime + ) + newUsageRecord.save() + usageRecordMapping[usageRecord]=newUsageRecord + + resourceInstanceMapping={} + from ...resources import ResourceInstance as NewResourceInstance + for resourceInstance in ResourceInstance.objects.filter(): + newResourceInstance = NewResourceInstance( + type=resourceInstance.type, + num=resourceInstance.num, + ownerElement=elementMapping(resourceInstance.ownerElement), + ownerElementId=elementMapping(resourceInstance.ownerElement).id, + ownerConnection=connectionMapping(resourceInstance.ownerConnection), + ownerConnectionId=connectionMapping(resourceInstance.ownerConnection).id, + attrs=resourceInstance.attrs + ) + - #TODO: Add connection(s) migration + #Update all elements to integrate the old references and relationships between them. + + for externalNetwork in External_Network.objects.filter(): + newExternalNetwork= elementMapping[externalNetwork] + newExternalNetwork.parent = elementMapping[externalNetwork.parent] + newExternalNetwork.parentId = elementMapping[externalNetwork.parent].id + newExternalNetwork.connection = connectionMapping[externalNetwork.connection] + newExternalNetwork.connectionId = connectionMapping[externalNetwork.connection].id + newExternalNetwork.usageStatistics = usageStatisticsMapping[externalNetwork.usageStatistic] + newExternalNetwork.usageStatisticsId = usageStatisticsMapping[externalNetwork.usageStatistic].id + newExternalNetwork.save() + + for kvmqm in KVMQM.objects.filter(): + newKVMQM = elementMapping[kvmqm] + newKVMQM.parent = elementMapping[kvmqm.parent] + newKVMQM.parentId = elementMapping[kvmqm.parent].id + newKVMQM.connection = connectionMapping[kvmqm.connection] + newKVMQM.connectionId = connectionMapping[kvmqm.connection].id + newKVMQM.usageStatistics = usageStatisticsMapping[kvmqm.usageStatistic] + newKVMQM.usageStatisticsId = usageStatisticsMapping[kvmqm.usageStatistic].id + newKVMQM.save() + + for kvmqm_interface in KVMQM_Interface.objects.filter(): + newKVMQM_Interface = elementMapping[kvmqm_interface] + newKVMQM_Interface.parent = elementMapping[kvmqm_interface.parent] + newKVMQM_Interface.parentId = elementMapping[kvmqm_interface.parent].id + newKVMQM_Interface.connection = connectionMapping[kvmqm_interface.connection] + newKVMQM_Interface.connectionId = connectionMapping[kvmqm_interface.connection].id + newKVMQM_Interface.usageStatistics = usageStatisticsMapping[kvmqm_interface.usageStatistic] + newKVMQM_Interface.usageStatisticsId = usageStatisticsMapping[kvmqm_interface.usageStatistic].id + newKVMQM_Interface.save() + + + for kvm in KVM.objects.filter(): + newKVM = elementMapping[kvm] + newKVM.parent = elementMapping[kvm.parent] + newKVM.parentId = elementMapping[kvm.parent].id + newKVM.connection = connectionMapping[kvm.connection] + newKVM.connectionId = connectionMapping[kvm.connection].id + newKVM.usageStatistics = usageStatisticsMapping[kvm.usageStatistic] + newKVM.usageStatisticsId = usageStatisticsMapping[kvm.usageStatistic].id + newKVM.save() + + for kvm_interface in KVM_Interface.objects.filter(): + newKVM_Interface = elementMapping[kvm_interface] + newKVM_Interface.parent = elementMapping[kvm_interface.parent] + newKVM_Interface.parentId = elementMapping[kvm_interface.parent].id + newKVM_Interface.connection = connectionMapping[kvm_interface.connection] + newKVM_Interface.connectionId = connectionMapping[kvm_interface.connection].id + newKVM_Interface.usageStatistics = usageStatisticsMapping[kvm_interface.usageStatistic] + newKVM_Interface.usageStatisticsId = usageStatisticsMapping[kvm_interface.usageStatistic].id + newKVM_Interface.save() + + for openvz in OpenVZ.objects.filter(): + newOpenVZ = elementMapping[openvz] + newOpenVZ.parent = elementMapping[openvz.parent] + newOpenVZ.parentId = elementMapping[openvz.parent].id + newOpenVZ.connection = connectionMapping[openvz.connection] + newOpenVZ.connectionId = connectionMapping[openvz.connection].id + newOpenVZ.usageStatistics = usageStatisticsMapping[openvz.usageStatistic] + newOpenVZ.usageStatisticsId = usageStatisticsMapping[openvz.usageStatistic].id + newOpenVZ.save() + + for openvz_interface in OpenVZ_Interface.objects.filter(): + newOpenVZ_Interface = elementMapping[openvz_interface] + newOpenVZ_Interface.parent = elementMapping[openvz_interface.parent] + newOpenVZ_Interface.parentId = elementMapping[openvz_interface.parent].id + newOpenVZ_Interface.connection = connectionMapping[openvz_interface.connection] + newOpenVZ_Interface.connectionId = connectionMapping[openvz_interface.connection].id + newOpenVZ_Interface.usageStatistics = usageStatisticsMapping[openvz_interface.usageStatistic] + newOpenVZ_Interface.usageStatisticsId = usageStatisticsMapping[openvz_interface.usageStatistic].id + newOpenVz_Interface.save() + + + for repy in Repy.objects.filter(): + newRepy = elementMapping[repy] + newRepy.parent = elementMapping[repy.parent] + newRepy.parentId = elementMapping[repy.parent].id + newRepy.connection = connectionMapping[repy.connection] + newRepy.connectionId = connectionMapping[repy.connection].id + newRepy.usageStatistics = usageStatisticsMapping[repy.usageStatistic] + newRepy.usageStatisticsId = usageStatisticsMapping[repy.usageStatistic].id + newRepy.save() + + for repy_interface in Repy_Interface.objects.filter(): + newRepy_Interface = elementMapping[repy_interface] + newRepy_Interface.parent = elementMapping[repy_interface.parent] + newRepy_Interface.parentId = elementMapping[repy_interface.parent].id + newRepy_Interface.connection = connectionMapping[repy_interface.connection] + newRepy_Interface.connectionId = connectionMapping[repy_interface.connection].id + newRepy_Interface.usageStatistics = usageStatisticsMapping[repy_interface.usageStatistic] + newRepy_Interface.usageStatisticsId = usageStatisticsMapping[repy_interface.usageStatistic].id + newRepy_Interface.save() + + for tinc in Tinc.objects.filter(): + newTinc = elementMapping[tinc] + newTinc.parent = elementMapping[tinc.parent] + newTinc.parentId = elementMapping[tinc.parent].id + newTinc.connection = connectionMapping[tinc.connection] + newTinc.connectionId = connectionMapping[tinc.connection].id + newTinc.usageStatistics = usageStatisticsMapping[tinc.usageStatistic] + newTinc.usageStatisticsId = usageStatisticsMapping[tinc.usageStatistic].id + newTinc.save() + + for udp_tunnel in UDP_Tunnel.objects.filter(): + newUDP_Tunnel = elementMapping[udp_tunnel] + newUDP_Tunnel.parent = elementMapping[udp_tunnel.parent] + newUDP_Tunnel.parentId = elementMapping[udp_tunnel.parent].id + newUDP_Tunnel.connection = connectionMapping[udp_tunnel.connection] + newUDP_Tunnel.connectionId = connectionMapping[udp_tunnel.connection].id + newUDP_Tunnel.usageStatistics = usageStatisticsMapping[udp_tunnel.usageStatistic] + newUDP_Tunnel.usageStatisticsId = usageStatisticsMapping[udp_tunnel.usageStatistic].id + newUDP_Tunnel.save() + + for vpncloud in VpnCloud.objects.filter(): + newVpnCloud = elementMapping[vpncloud] + newVpnCloud.parent = elementMapping[vpncloud.parent] + newVpnCloud.parentId = elementMapping[vpncloud.parent].id + newVpnCloud.connection = connectionMapping[vpncloud.connection] + newVpnCloud.connectionId = connectionMapping[vpncloud.connection].id + newVpnCloud.usageStatistics = usageStatisticsMapping[vpncloud.usageStatistic] + newVpnCloud.usageStatisticsId = usageStatisticsMapping[vpncloud.usageStatistic].id + newVpnCloud.save() + + for bridge in Bridge.objects.filter(): + newBridge = connectionMapping[bridge] + newBridge.usageStatistics = usageStatisticsMapping[bridge.usageStatistic] + newBridge.usageStatisticsId = usageStatisticsMapping[bridge.usageStatistic].id + + eleList=[] + for element in bridge.elements: + eleList.append(elementMapping[element]) + newBridge.elements = eleList + newBridge.save() + + for fixed_bridge in Fixed_Bridge.objects.filter(): + newFixedBridge = connectionMapping[fixed_bridge] + newFixedBridge.usageStatistics = usageStatisticsMapping[fixed_bridge.usageStatistic] + newFixedBridge.usageStatisticsId = usageStatisticsMapping[fixed_bridge.usageStatistic].id + + eleList = [] + for element in fixed_bridge.elements: + eleList.append(elementMapping[element]) + newFixedBridge.elements = eleList + newFixedBridge.save() + + + + #TODO: Add connection(s) migration #Test everythin once #Add correct referencing of new objects for mongodb \ No newline at end of file From cce09a0cd46c85e4d16ab26d5e476376a5a5909c Mon Sep 17 00:00:00 2001 From: Sebastian Wuest Date: Tue, 13 Jun 2017 14:34:59 +0200 Subject: [PATCH 6/7] A lot of bugfixes for the migration script. Still not stable. --- backend_core/tomato/host/element.py | 3 +- hostmanager/tomato/accounting.py | 4 +- hostmanager/tomato/api/accounting.py | 4 +- hostmanager/tomato/api/connections.py | 8 + hostmanager/tomato/api/elements.py | 8 + hostmanager/tomato/api/resources.py | 8 + hostmanager/tomato/connections/__init__.py | 18 +- hostmanager/tomato/connections/bridge.py | 4 +- .../tomato/db/migrations/migration_0001.py | 857 ++++++------------ hostmanager/tomato/elements/__init__.py | 18 +- hostmanager/tomato/elements/kvm.py | 2 - hostmanager/tomato/elements/kvmqm.py | 2 - hostmanager/tomato/resources/__init__.py | 51 +- 13 files changed, 371 insertions(+), 616 deletions(-) diff --git a/backend_core/tomato/host/element.py b/backend_core/tomato/host/element.py index bd8a81fd0..a3c074ed7 100644 --- a/backend_core/tomato/host/element.py +++ b/backend_core/tomato/host/element.py @@ -125,6 +125,7 @@ def synchronize(self): if (not self.topologyElement and not self.topologyConnection) or (self.host is None): self.remove() return + self.updateInfo() self.modify(timeout=time.time() + settings.get_host_connections_settings()['component-timeout']) except error.UserError, err: if err.code == error.UserError.ENTITY_DOES_NOT_EXIST: @@ -144,6 +145,6 @@ def synchronize(id_): try: HostElement.objects.get(id=id_).synchronize() except DoesNotExist: - pass # nothong to synchronize + pass # nothing to synchronize scheduler.scheduleMaintenance(min(3600, settings.get_host_connections_settings()['component-timeout']), list, synchronize) \ No newline at end of file diff --git a/hostmanager/tomato/accounting.py b/hostmanager/tomato/accounting.py index ece481213..90b6db847 100644 --- a/hostmanager/tomato/accounting.py +++ b/hostmanager/tomato/accounting.py @@ -104,8 +104,8 @@ class UsageStatistics(BaseDocument): ATTRIBUTES = { "id": IdAttribute(), "begin": Attribute(field=begin, schema=schema.Number()), - "element": Attribute(get=lambda self: self.element.id if self.element else None), - "connection": Attribute(get=lambda self: self.connection.id if self.element else None) + "element": Attribute(get=lambda self: self.element.getId() if self.element else None), + "connection": Attribute(get=lambda self: self.connection.getId() if self.connection else None) } meta = { diff --git a/hostmanager/tomato/api/accounting.py b/hostmanager/tomato/api/accounting.py index 1cf6b9d96..c71d54f8e 100644 --- a/hostmanager/tomato/api/accounting.py +++ b/hostmanager/tomato/api/accounting.py @@ -54,8 +54,8 @@ def accounting_statistics(type=None, after=None, before=None): #@ReservedAssignm are the connection ids (as strings) and the values are the usage statistics. """ - elSt = dict([(str(el.id), el.getUsageStatistics().info(type, after, before)) for el in elements.getAll(owner=currentUser())]) - conSt = dict([(str(con.id), con.getUsageStatistics().info(type, after, before)) for con in connections.getAll(owner=currentUser())]) + elSt = dict([(str(el.getId()), el.getUsageStatistics().info(type, after, before)) for el in elements.getAll(owner=currentUser())]) + conSt = dict([(str(con.getId()), con.getUsageStatistics().info(type, after, before)) for con in connections.getAll(owner=currentUser())]) return {"elements": elSt, "connections": conSt} def accounting_element_statistics(id, type=None, after=None, before=None): #@ReservedAssignment diff --git a/hostmanager/tomato/api/connections.py b/hostmanager/tomato/api/connections.py index 08cb968a8..1ff76d013 100644 --- a/hostmanager/tomato/api/connections.py +++ b/hostmanager/tomato/api/connections.py @@ -16,6 +16,14 @@ # along with this program. If not, see def _getConnection(id_): + import sys + try: + id_ = str(id_) + if sys.getsizeof(id_) < 60: + oldId=int(str(id_)) + id_ = "{0:0{1}x}".format(oldId, 24) + except: + pass con = connections.get(id_, owner=currentUser()) UserError.check(con, UserError.ENTITY_DOES_NOT_EXIST, "No such connection", data={"id": id_}) return con diff --git a/hostmanager/tomato/api/elements.py b/hostmanager/tomato/api/elements.py index 0a2764b82..b6a0b41bd 100644 --- a/hostmanager/tomato/api/elements.py +++ b/hostmanager/tomato/api/elements.py @@ -16,6 +16,14 @@ # along with this program. If not, see def _getElement(id_): + import sys + try: + id_ = str(id_) + if sys.getsizeof(id_) < 60: + oldId=int(str(id_)) + id_ = "{0:0{1}x}".format(oldId, 24) + except: + pass el = elements.get(id_, owner=currentUser()) UserError.check(el, UserError.ENTITY_DOES_NOT_EXIST, "No such element", data={"id": id_}) return el diff --git a/hostmanager/tomato/api/resources.py b/hostmanager/tomato/api/resources.py index ab0b293e7..8092bf2a9 100644 --- a/hostmanager/tomato/api/resources.py +++ b/hostmanager/tomato/api/resources.py @@ -16,6 +16,14 @@ # along with this program. If not, see def _getResource(id_): + import sys + try: + id_ = str(id_) + if sys.getsizeof(id_) < 60: + oldId=int(str(id_)) + id_ = "{0:0{1}x}".format(oldId, 24) + except: + pass res = resources.get(id_) UserError.check(res, UserError.ENTITY_DOES_NOT_EXIST, "No such resource", data={"id": id_}) return res diff --git a/hostmanager/tomato/connections/__init__.py b/hostmanager/tomato/connections/__init__.py index c34ba3868..b09763de5 100644 --- a/hostmanager/tomato/connections/__init__.py +++ b/hostmanager/tomato/connections/__init__.py @@ -156,6 +156,20 @@ class that it possesses. Due to a limitation of the database backend, pass raise InternalError(message="Failed to cast connection", code=InternalError.UPCAST, data={"id": str(self.id), "type": self.type}) + def getId(self): + """ + Returns the element id, converts automatically back to old postgres IDs if possible + :return: String + """ + id_ = str(self.id) + try: + #As we converted old ids by adding zeros to the left, we can check if the first 12 characters are zeros + if id_[0:12] == "000000000000": + id_ = int(id_, 16) + except: + pass + return id_ + def dataPath(self, filename=""): """ This method can be used to create filenames relative to a directory @@ -170,7 +184,7 @@ def dataPath(self, filename=""): @param filename: a filename relative to the data path @type filename: str """ - return os.path.join(config.DATA_DIR, self.TYPE, str(self.id), filename) + return os.path.join(config.DATA_DIR, self.TYPE, str(self.getId()), filename) @classmethod def determineConcept(cls, el1, el2): @@ -274,7 +288,7 @@ def getElements(self): def info(self): els = [str(el.id) for el in self.elements] return { - "id": str(self.id), + "id": str(self.getId()), "type": self.type, "state": self.state, "attrs": Entity.info(self), diff --git a/hostmanager/tomato/connections/bridge.py b/hostmanager/tomato/connections/bridge.py index 6bddee00e..9682db539 100644 --- a/hostmanager/tomato/connections/bridge.py +++ b/hostmanager/tomato/connections/bridge.py @@ -73,8 +73,9 @@ def type(self): def init(self, *args, **kwargs): self.state = StateName.CREATED connections.Connection.init(self, *args, **kwargs) #no id and no attrs before this line - self.bridge = "br%s" % str(self.id)[13:24] + self.bridge = "br%s" % (str(self.getId())[13:24] if type(self.getId()) == type(str()) else str(self.getId())) self.capture_port = self.getResource("port") + self.update_or_save(bridge=self.bridge, capture_port=self.capture_port) def _startCapturing(self): if not self.capturing or self.state == StateName.CREATED: @@ -245,6 +246,7 @@ def action_stop(self): if net.bridgeExists(self.bridge): net.ifDown(self.bridge) net.bridgeRemove(self.bridge) + self.update_or_save(bridge=self.bridge, capture_port=self.capture_port) self.setState(StateName.CREATED) def remove(self): diff --git a/hostmanager/tomato/db/migrations/migration_0001.py b/hostmanager/tomato/db/migrations/migration_0001.py index d39842fa2..c31932d89 100644 --- a/hostmanager/tomato/db/migrations/migration_0001.py +++ b/hostmanager/tomato/db/migrations/migration_0001.py @@ -1,359 +1,7 @@ - -from django.db import models -from ...lib import db, attributes, logging -from ...lib.attributes import Attr -from ...lib.constants import ActionName, StateName, TechName - -kblang_options = {"en-us": "English (US)", - "en-gb": "English (GB)", - "de": "German", - "fr": "French", - "ja": "Japanese" -} -class User(models.Model): - name = models.CharField(max_length=255, unique=True) - -class UsageStatistics(attributes.Mixin, models.Model): - begin = models.FloatField() #unix timestamp - #records: [UsageRecord] - attrs = db.JSONField() - -class UsageRecord(models.Model): - statistics = models.ForeignKey(UsageStatistics, related_name="records") - type = models.CharField(max_length=10, choices=[(t, t) for t in ["single", "5minutes", "hour", "day", "month", "year"]]) #@ReservedAssignment - begin = models.FloatField() #unix timestamp - end = models.FloatField() #unix timestamp - measurements = models.IntegerField() - #using fields to save space - memory = models.FloatField() #unit: bytes - diskspace = models.FloatField() #unit: bytes - traffic = models.FloatField() #unit: bytes - cputime = models.FloatField() #unit: cpu seconds - - -class Resource(db.ChangesetMixin, attributes.Mixin, models.Model): - type = models.CharField(max_length=20, validators=[db.nameValidator], choices=[(t, t) for t in ['template','network']]) #@ReservedAssignment - attrs = db.JSONField() - -class ResourceInstance(db.ChangesetMixin, attributes.Mixin, models.Model): - type = models.CharField(max_length=20, validators=[db.nameValidator], choices=[(t, t) for t in ['template','network']]) #@ReservedAssignment - num = models.IntegerField() - ownerElement = models.ForeignKey(Element, null=True) - ownerConnection = models.ForeignKey(Connection, null=True) - attrs = db.JSONField() - -class Network(Resource): - owner = models.ForeignKey(User, related_name='networks') - kind = models.CharField(max_length=50) - bridge = models.CharField(max_length=20) - preference = models.IntegerField(default=0) - -class Template(Resource): - owner = models.ForeignKey(User, related_name='templates') - tech = models.CharField(max_length=20) - name = models.CharField(max_length=50) - preference = models.IntegerField(default=0) - urls = attributes.attribute("urls", list) - checksum = attributes.attribute("checksum", str) - size = attributes.attribute("size", long) - popularity = attributes.attribute("popularity", float) - ready = attributes.attribute("ready", bool) - kblang = attributes.attribute("kblang", str, null=False, default="en-us") - - -class Connection(db.ChangesetMixin, attributes.Mixin, models.Model): - type = models.CharField(max_length=20, validators=[db.nameValidator], - choices=[(t, t) for t in ["bridge", "fixed_bridge"]]) # @ReservedAssignment - owner = models.ForeignKey(User, related_name='connections') - state = models.CharField(max_length=20, validators=[db.nameValidator]) - usageStatistics = models.OneToOneField(UsageStatistics, null=True, related_name='connection') - attrs = db.JSONField() - # elements: set of elements.Element - - -class Bridge(Connection): - bridge_attr = Attr("bridge", type="str") - bridge = bridge_attr.attribute() - - emulation_attr = Attr("emulation", desc="Enable emulation", type="bool", default=True) - emulation = emulation_attr.attribute() - - bandwidth_to_attr = Attr("bandwidth_to", desc="Bandwidth", unit="kbit/s", type="float", minValue=0, - maxValue=1000000, default=10000) - bandwidth_to = bandwidth_to_attr.attribute() - bandwidth_from_attr = Attr("bandwidth_from", desc="Bandwidth", unit="kbit/s", type="float", minValue=0, - maxValue=1000000, default=10000) - bandwidth_from = bandwidth_from_attr.attribute() - - lossratio_to_attr = Attr("lossratio_to", desc="Loss ratio", unit="%", type="float", minValue=0.0, maxValue=100.0, - default=0.0) - lossratio_to = lossratio_to_attr.attribute() - lossratio_from_attr = Attr("lossratio_from", desc="Loss ratio", unit="%", type="float", minValue=0.0, - maxValue=100.0, default=0.0) - lossratio_from = lossratio_from_attr.attribute() - - duplicate_to_attr = Attr("duplicate_to", desc="Duplication ratio", unit="%", type="float", minValue=0.0, - maxValue=100.0, default=0.0) - duplicate_to = duplicate_to_attr.attribute() - duplicate_from_attr = Attr("duplicate_from", desc="Duplication ratio", unit="%", type="float", minValue=0.0, - maxValue=100.0, default=0.0) - duplicate_from = duplicate_from_attr.attribute() - - corrupt_to_attr = Attr("corrupt_to", desc="Corruption ratio", unit="%", type="float", minValue=0.0, maxValue=100.0, - default=0.0) - corrupt_to = corrupt_to_attr.attribute() - corrupt_from_attr = Attr("corrupt_from", desc="Corruption ratio", unit="%", type="float", minValue=0.0, - maxValue=100.0, default=0.0) - corrupt_from = corrupt_from_attr.attribute() - - delay_to_attr = Attr("delay_to", desc="Delay", unit="ms", type="float", minValue=0.0, default=0.0) - delay_to = delay_to_attr.attribute() - delay_from_attr = Attr("delay_from", desc="Delay", unit="ms", type="float", minValue=0.0, default=0.0) - delay_from = delay_from_attr.attribute() - - jitter_to_attr = Attr("jitter_to", desc="Jitter", unit="ms", type="float", minValue=0.0, default=0.0) - jitter_to = jitter_to_attr.attribute() - jitter_from_attr = Attr("jitter_from", desc="Jitter", unit="ms", type="float", minValue=0.0, default=0.0) - jitter_from = jitter_from_attr.attribute() - - distribution_to_attr = Attr("distribution_to", desc="Distribution", type="str", - options={"uniform": "Uniform", "normal": "Normal", "pareto": "Pareto", - "paretonormal": "Pareto-Normal"}, default="uniform") - distribution_to = distribution_to_attr.attribute() - distribution_from_attr = Attr("distribution_from", desc="Distribution", type="str", - options={"uniform": "Uniform", "normal": "Normal", "pareto": "Pareto", - "paretonormal": "Pareto-Normal"}, default="uniform") - distribution_from = distribution_from_attr.attribute() - - capturing_attr = Attr("capturing", desc="Enable packet capturing", type="bool", default=False) - capturing = capturing_attr.attribute() - capture_filter_attr = Attr("capture_filter", desc="Packet filter expression", type="str", default="") - capture_filter = capture_filter_attr.attribute() - capture_port_attr = Attr("capture_port", type="int") - capture_port = capture_port_attr.attribute() - capture_mode_attr = Attr("capture_mode", desc="Capture mode", type="str", - options={"net": "Via network", "file": "For download"}, default="file") - capture_mode = capture_mode_attr.attribute() - capture_pid_attr = Attr("capture_pid", type="int") - capture_pid = capture_pid_attr.attribute() - - TYPE = "bridge" - -class Fixed_Bridge(Connection): - TYPE = "fixed_bridge" - -ELEMENT_TYPES= ["kvm", "kvm_interface", - "kvmqm","kvmqm_interface", - "openvz","openvz_interface", - "lxc", "lxc_interface", - "repy","repy_interface", - "external_network", - "external_network_endpoint", - "tinc","tinc_vpn","tinc_endpoint", - "udp_endpoint","udp_tunnel", - "vpncloud","vpncloud_endpoint"] - -class Element(db.ChangesetMixin, attributes.Mixin, models.Model): - type = models.CharField(max_length=20, validators=[db.nameValidator], - choices=[(t, t) for t in ELEMENT_TYPES]) # @ReservedAssignment - owner = models.ForeignKey(User, related_name='elements') - parent = models.ForeignKey('self', null=True, related_name='children') - connection = models.ForeignKey(Connection, null=True, related_name='elements') - usageStatistics = models.OneToOneField(UsageStatistics, null=True, related_name='element') - state = models.CharField(max_length=20, validators=[db.nameValidator]) - timeout = models.FloatField() - timeout_attr = Attr("timeout", desc="Timeout", states=[], type="float", null=False) - attrs = db.JSONField() - -class RexTFVElement: - rextfv_max_size = None - -class External_Network(Element): - network_attr = Attr("network", null=True) - network = models.ForeignKey(Network, null=True, related_name="instances") - -class KVM(RexTFVElement,Element): - vmid_attr = Attr("vmid", type="int") - vmid = vmid_attr.attribute() - websocket_port_attr = Attr("websocket_port", type="int") - websocket_port = websocket_port_attr.attribute() - websocket_pid_attr = Attr("websocket_pid", type="int") - websocket_pid = websocket_pid_attr.attribute() - vncport_attr = Attr("vncport", type="int") - vncport = vncport_attr.attribute() - vncpid_attr = Attr("vncpid", type="int") - vncpid = vncpid_attr.attribute() - vncpassword_attr = Attr("vncpassword", type="str") - vncpassword = vncpassword_attr.attribute() - cpus_attr = Attr("cpus", desc="Number of CPUs", states=[StateName.CREATED, StateName.PREPARED], type="int", minValue=1, maxValue=4, default=1) - cpus = cpus_attr.attribute() - ram_attr = Attr("ram", desc="RAM", unit="MB", states=[StateName.CREATED, StateName.PREPARED], type="int", minValue=64, maxValue=8192, default=256) - ram = ram_attr.attribute() - kblang_attr = Attr("kblang", desc="Keyboard language", states=[StateName.CREATED, StateName.PREPARED], type="str", options=kblang_options, default=None, null=True) - #["pt", "tr", "ja", "es", "no", "is", "fr-ca", "fr", "pt-br", "da", "fr-ch", "sl", "de-ch", "en-gb", "it", "en-us", "fr-be", "hu", "pl", "nl", "mk", "fi", "lt", "sv", "de"] - kblang = kblang_attr.attribute() - usbtablet_attr = Attr("usbtablet", desc="USB tablet mouse mode", states=[StateName.CREATED, StateName.PREPARED], type="bool", default=True) - usbtablet = usbtablet_attr.attribute() - template_attr = Attr("template", desc="Template", states=[StateName.CREATED, StateName.PREPARED], type="str", null=True) - template = models.ForeignKey(Template, null=True) - -class KVM_Interface(Element): - num_attr = Attr("num", type="int") - num = num_attr.attribute() - name_attr = Attr("name", desc="Name", type="str", regExp="^eth[0-9]+$", states=[StateName.CREATED]) - mac_attr = Attr("mac", desc="MAC Address", type="str") - mac = mac_attr.attribute() - ipspy_pid_attr = Attr("ipspy_pid", type="int") - ipspy_pid = ipspy_pid_attr.attribute() - used_addresses_attr = Attr("used_addresses", type=list, default=[]) - used_addresses = used_addresses_attr.attribute() - -class KVMQM(RexTFVElement,Element): - vmid_attr = Attr("vmid", type="int") - vmid = vmid_attr.attribute() - websocket_port_attr = Attr("websocket_port", type="int") - websocket_port = websocket_port_attr.attribute() - websocket_pid_attr = Attr("websocket_pid", type="int") - websocket_pid = websocket_pid_attr.attribute() - vncport_attr = Attr("vncport", type="int") - vncport = vncport_attr.attribute() - vncpid_attr = Attr("vncpid", type="int") - vncpid = vncpid_attr.attribute() - vncpassword_attr = Attr("vncpassword", type="str") - vncpassword = vncpassword_attr.attribute() - cpus_attr = Attr("cpus", desc="Number of CPUs", states=[StateName.CREATED, StateName.PREPARED], type="int", minValue=1, maxValue=4, default=1) - cpus = cpus_attr.attribute() - ram_attr = Attr("ram", desc="RAM", unit="MB", states=[StateName.CREATED, StateName.PREPARED], type="int", minValue=64, maxValue=8192, default=256) - ram = ram_attr.attribute() - kblang_attr = Attr("kblang", desc="Keyboard language", states=[StateName.CREATED, StateName.PREPARED], type="str", options=kblang_options, default=None, null=True) - #["pt", "tr", "ja", "es", "no", "is", "fr-ca", "fr", "pt-br", "da", "fr-ch", "sl", "de-ch", "en-gb", "it", "en-us", "fr-be", "hu", "pl", "nl", "mk", "fi", "lt", "sv", "de"] - kblang = kblang_attr.attribute() - usbtablet_attr = Attr("usbtablet", desc="USB tablet mouse mode", states=[StateName.CREATED, StateName.PREPARED], type="bool", default=True) - usbtablet = usbtablet_attr.attribute() - template_attr = Attr("template", desc="Template", states=[StateName.CREATED, StateName.PREPARED], type="str", null=True) - template = models.ForeignKey(Template, null=True) - -class KVMQM_Interface(Element): - num_attr = Attr("num", type="int") - num = num_attr.attribute() - name_attr = Attr("name", desc="Name", type="str", regExp="^eth[0-9]+$", states=[StateName.CREATED]) - mac_attr = Attr("mac", desc="MAC Address", type="str") - mac = mac_attr.attribute() - ipspy_pid_attr = Attr("ipspy_pid", type="int") - ipspy_pid = ipspy_pid_attr.attribute() - used_addresses_attr = Attr("used_addresses", type=list, default=[]) - used_addresses = used_addresses_attr.attribute() - -class OpenVZ(RexTFVElement,Element): - vmid_attr = Attr("vmid", type="int") - vmid = vmid_attr.attribute() - websocket_port_attr = Attr("websocket_port", type="int") - websocket_port = websocket_port_attr.attribute() - websocket_pid_attr = Attr("websocket_pid", type="int") - websocket_pid = websocket_pid_attr.attribute() - vncport_attr = Attr("vncport", type="int") - vncport = vncport_attr.attribute() - vncpid_attr = Attr("vncpid", type="int") - vncpid = vncpid_attr.attribute() - vncpassword_attr = Attr("vncpassword", type="str") - vncpassword = vncpassword_attr.attribute() - ram_attr = Attr("ram", desc="RAM", unit="MB", type="int", minValue=64, maxValue=4096, default=256) - ram = ram_attr.attribute() - cpus_attr = Attr("cpus", desc="Number of CPUs", type="float", minValue=0.1, maxValue=4.0, default=1.0) - cpus = cpus_attr.attribute() - diskspace_attr = Attr("diskspace", desc="Disk space", unit="MB", type="int", minValue=512, maxValue=102400, default=10240) - diskspace = diskspace_attr.attribute() - rootpassword_attr = Attr("rootpassword", desc="Root password", type="str") - rootpassword = rootpassword_attr.attribute() - hostname_attr = Attr("hostname", desc="Hostname", type="str") - hostname = hostname_attr.attribute() - gateway4_attr = Attr("gateway4", desc="IPv4 gateway", type="str") - gateway4 = gateway4_attr.attribute() - gateway6_attr = Attr("gateway6", desc="IPv6 gateway", type="str") - gateway6 = gateway6_attr.attribute() - template_attr = Attr("template", desc="Template", states=[StateName.CREATED, StateName.PREPARED], type="str", null=True) - template = models.ForeignKey(Template, null=True) - -class OpenVZ_Interface(Element): - name_attr = Attr("name", desc="Name", type="str", regExp="^eth[0-9]+$", states=[StateName.CREATED]) - name = name_attr.attribute() - ip4address_attr = Attr("ip4address", desc="IPv4 address", type="str") - ip4address = ip4address_attr.attribute() - ip6address_attr = Attr("ip6address", desc="IPv6 address", type="str") - ip6address = ip6address_attr.attribute() - use_dhcp_attr = Attr("use_dhcp", desc="Use DHCP", type="bool", default=False) - use_dhcp = use_dhcp_attr.attribute() - mac_attr = Attr("mac", desc="MAC Address", type="str") - mac = mac_attr.attribute() - ipspy_pid_attr = Attr("ipspy_pid", type="int") - ipspy_pid = ipspy_pid_attr.attribute() - used_addresses_attr = Attr("used_addresses", type=list, default=[]) - used_addresses = used_addresses_attr.attribute() - -class Repy(Element): - pid_attr = Attr("pid", type="int") - pid = pid_attr.attribute() - websocket_port_attr = Attr("websocket_port", type="int") - websocket_port = websocket_port_attr.attribute() - websocket_pid_attr = Attr("websocket_pid", type="int") - websocket_pid = websocket_pid_attr.attribute() - vncport_attr = Attr("vncport", type="int") - vncport = vncport_attr.attribute() - vncpid_attr = Attr("vncpid", type="int") - vncpid = vncpid_attr.attribute() - vncpassword_attr = Attr("vncpassword", type="str") - vncpassword = vncpassword_attr.attribute() - args_attr = Attr("args", desc="Arguments", states=[StateName.PREPARED], default=[]) - args = args_attr.attribute() - cpus_attr = Attr("cpus", desc="Number of CPUs", states=[StateName.PREPARED], type="float", minValue=0.01, maxValue=4.0, default=0.25) - cpus = cpus_attr.attribute() - ram_attr = Attr("ram", desc="RAM", unit="MB", states=[StateName.PREPARED], type="int", minValue=10, maxValue=4096, default=25) - ram = ram_attr.attribute() - bandwidth_attr = Attr("bandwidth", desc="Bandwidth", unit="bytes/s", states=[StateName.PREPARED], type="int", minValue=1024, maxValue=10000000000, default=1000000) - bandwidth = bandwidth_attr.attribute() - #TODO: use template ref instead of attr - template_attr = Attr("template", desc="Template", states=[StateName.PREPARED], type="str", null=True) - template = models.ForeignKey(Template, null=True) - -class Repy_Interface(Element): - name_attr = Attr("name", desc="Name", type="str", regExp="^eth[0-9]+$") - name = name_attr.attribute() - ipspy_pid_attr = Attr("ipspy_pid", type="int") - ipspy_pid = ipspy_pid_attr.attribute() - used_addresses_attr = Attr("used_addresses", type=list, default=[]) - used_addresses = used_addresses_attr.attribute() - -class Tinc(Element): - port_attr = Attr("port", type="int") - port = port_attr.attribute() - path_attr = Attr("path") - path = path_attr.attribute() - mode_attr = Attr("mode", desc="Mode", states=[StateName.CREATED], options={"hub": "Hub (broadcast)", "switch": "Switch (learning)"}, default="switch") - mode = mode_attr.attribute() - privkey_attr = Attr("privkey", desc="Private key") - privkey = privkey_attr.attribute() - pubkey_attr = Attr("pubkey", desc="Public key") - pubkey = pubkey_attr.attribute() - peers_attr = Attr("peers", desc="Peers", states=[StateName.CREATED], default=[]) - peers = peers_attr.attribute() - -class UDP_Tunnel(Element): - pid_attr = Attr("pid", type="int") - pid = pid_attr.attribute() - port_attr = Attr("port", type="int") - port = port_attr.attribute() - connect_attr = Attr("connect", desc="Connect to", states=[StateName.CREATED], type="str", null=True, default=None) - connect = connect_attr.attribute() - -class VpnCloud(Element): - port_attr = Attr("port", type="int") - port = port_attr.attribute() - pid_attr = Attr("pid", type="int", null=True) - pid = pid_attr.attribute() - network_id_attr = Attr("network_id", type="int") - network_id = network_id_attr.attribute() - peers_attr = Attr("peers", desc="Peers", states=[StateName.CREATED], default=[]) - peers = peers_attr.attribute() +from ...lib.newcmd.util import run, CommandError, cmd, proc +from ... import config +import os +from ...modelsold import User, Resource, Template, Network, Element, External_Network, KVM, KVM_Interface, KVMQM, KVMQM_Interface, OpenVZ, OpenVZ_Interface, Repy, Repy_Interface, Tinc, UDP_Tunnel, VpnCloud, Connection, Bridge, Fixed_Bridge, UsageRecord, UsageStatistics, ResourceInstance ''' Migration through 2 Steps: @@ -364,29 +12,36 @@ class VpnCloud(Element): def migrate(): #Create old elements in the new database with all attributes except for references + from django.core.management import call_command + call_command('syncdb', verbosity=0) userMapping = {} from ...user import User as NewUser for user in User.objects.filter(): - newUser = NewUser(name=user.name) + newUser = NewUser( + id="{0:0{1}x}".format(user.id, 24), + name=user.name) newUser.save() - userMapping[user]=newUser + userMapping[user.id]=newUser resourceMapping = {} from ...resources import Resource as NewResource for resource in Resource.objects.filter(): if resource.type not in ["network","template"]: - newResource = NewResource(attrs=resource.attrs, - type=resource.type) + newResource = NewResource( + id="{0:0{1}x}".format(resource.id, 24), + attrs=resource.attrs, + type=resource.type) newResource.save() - resourceMapping[resource] = newResource + resourceMapping[resource.id] = newResource from ...resources.template import Template as NewTemplate for template in Template.objects.filter(): - newTemplate = NewTemplate(attrs=resource.attrs, + newTemplate = NewTemplate( + id="{0:0{1}x}".format(template.id, 24), + attrs=resource.attrs, type=resource.type, - owner=userMapping[template.owner], - ownerId=userMapping[template.owner].id, + owner=userMapping[template.owner.id], tech=template.tech, name=template.name, preference=template.preference, @@ -397,77 +52,78 @@ def migrate(): ready=template.ready, kblang=template.kblang) newTemplate.save() - resourceMapping[template] = newTemplate + resourceMapping[template.id] = newTemplate from ...resources.network import Network as NewNetwork for network in Network.objects.filter(): - newNetwork = NewNetwork(attrs=resource.attrs, + newNetwork = NewNetwork( + id="{0:0{1}x}".format(network.id, 24), + attrs=resource.attrs, type=resource.type, - owner=userMapping[network.owner], - ownerId=userMapping[network.owner].id, + owner=userMapping[network.owner.id], kind=network.kind, bridge=network.bridge, preference=network.preference) - resourceMapping[network] = newNetwork + resourceMapping[network.id] = newNetwork elementMapping = {} from ...elements.external_network import External_Network as NewExternalNetwork for externalNetwork in External_Network.objects.filter(): newExternalNetwork = NewExternalNetwork( - owner=userMapping[externalNetwork.owner], - ownerId = userMapping[externalNetwork.owner].id, + id="{0:0{1}x}".format(externalNetwork.id, 24), + owner=userMapping[externalNetwork.owner.id], state=externalNetwork.state, timeout=externalNetwork.timeout, - network=resourceMapping[externalNetwork.network], - networkId=resourceMapping[externalNetwork.network].id + network=resourceMapping[externalNetwork.network.id] ) - elementMapping[externalNetwork] = newExternalNetwork + elementMapping[externalNetwork.id] = newExternalNetwork from ...elements.kvmqm import KVMQM as New_KVMQM for kvmqm in KVMQM.objects.filter(): newKVMQM = New_KVMQM( - owner=userMapping[kvmqm.owner], - ownerId = userMapping[kvmqm.owner].id, + id="{0:0{1}x}".format(kvmqm.id, 24), + owner=userMapping[kvmqm.owner.id], state=kvmqm.state, timeout=kvmqm.timeout, - vmid = kvmqm.vmid, - websocket_port = kvmqm.websocket_port, - websocket_pid = kvmqm.websocket_pid, - vncport = kvmqm.vncport, - vncpid = kvmqm.vncpid, - vncpassword = kvmqm.vncpassword, - cpus = kvmqm.cpus, - ram = kvmqm.ram, - kblang = kvmqm.kblang, - usbtablet = kvmqm.usbtablet, - template = resourceMapping[kvmqm.template], - templateId = resourceMapping[kvmqm.template].id + vmid=kvmqm.vmid, + websocket_port=kvmqm.websocket_port, + websocket_pid=kvmqm.websocket_pid, + vncport=kvmqm.vncport, + vncpid=kvmqm.vncpid, + vncpassword=kvmqm.vncpassword, + cpus=kvmqm.cpus, + ram=kvmqm.ram, + kblang=kvmqm.kblang, + usbtablet=kvmqm.usbtablet, + template=resourceMapping[kvmqm.template.id] ) newKVMQM.save() - elementMapping[kvmqm] = newKVMQM + elementMapping[kvmqm.id] = newKVMQM - from ...elements.kvmqm import KVMQM_Interface as KVQM_Interface - for kvmqm_interface in KVMQM_Interface.objects.filter(): - newKVMQMInterface = KVQM_Interface( - owner=userMapping[kvmqm_interface.owner], - ownerId=userMapping[kvmqm_interface.owner].id, + from ...elements.kvmqm import KVMQM_Interface as New_KVMQM_Interface + for kvmqm_interface in KVM_Interface.objects.filter(): #This is due to an old naming error (Should have been KVMQM_Interface) + newKVMQMInterface = New_KVMQM_Interface( + id="{0:0{1}x}".format(kvmqm_interface.id, 24), + owner=userMapping[kvmqm_interface.owner.id], state=kvmqm_interface.state, timeout=kvmqm_interface.timeout, num=kvmqm_interface.num, - name=kvmqm_interface.name, mac=kvmqm_interface.mac, - ipspy_pid=kvmqm_interface.ipspy_id, + ipspy_pid=kvmqm_interface.ipspy_pid, used_addresses=kvmqm_interface.used_addresses ) newKVMQMInterface.save() - elementMapping[kvmqm_interface] = newKVMQMInterface + elementMapping[kvmqm_interface.id] = newKVMQMInterface + + #They should never exist + """ from ...elements.kvm import KVM as New_KVM for kvm in KVM.objects.filter(): newKVM = New_KVM( - owner=userMapping[kvm.owner], - ownerId = userMapping[kvm.owner].id, + id="{0:0{1}x}".format(kvm.id, 24), + owner=userMapping[kvm.owner.id], state=kvm.state, timeout=kvm.timeout, vmid=kvm.vmid, @@ -480,59 +136,57 @@ def migrate(): ram=kvm.ram, kblang=kvm.kblang, usbtablet=kvm.usbtablet, - template = resourceMapping[kvm.template], - templateId = resourceMapping[kvm.template].id + template = resourceMapping[kvm.template.id] ) newKVM.save() - elementMapping[kvm] = newKVM - - from ...elements.kvm import KVM_Interface as KV_Interface - for kvm_interface in KVM.objects.filter(): - newKVMInterface = KV_Interface( - owner=userMapping[kvm_interface.owner], - ownerId=userMapping[kvm_interface.owner].id, + elementMapping[kvm.id] = newKVM + + from ...elements.kvm import KVM_Interface as New_KVM_Interface + for kvm_interface in KVM_Interface.objects.filter(): + newKVMInterface = New_KVM_Interface( + id="{0:0{1}x}".format(kvm_interface.id, 24), + owner=userMapping[kvm_interface.owner.id], state=kvm_interface.state, timeout=kvm_interface.timeout, num=kvm_interface.num, - name=kvm_interface.name, mac=kvm_interface.mac, - ipspy_pid=kvm_interface.ipspy_id, + ipspy_pid=kvm_interface.ipspy_pid, used_addresses=kvm_interface.used_addresses ) newKVMInterface.save() - elementMapping[kvm_interface] = newKVMInterface + elementMapping[kvm_interface.id] = newKVMInterface + """ from ...elements.openvz import OpenVZ as New_OpenVZ for openvz in OpenVZ.objects.filter(): newOpenVZ = New_OpenVZ( - owner=userMapping[openvz.owner], - ownerId=userMapping[openvz.owner].id, + id="{0:0{1}x}".format(openvz.id, 24), + owner=userMapping[openvz.owner.id], state=openvz.state, timeout=openvz.timeout, vmid=openvz.vmid, - websocket_port = openvz.websocket_port, - websocket_pid = openvz.websocket_pid, - vncport = openvz.vncport, - vncpid = openvz.vncpid, - vncpassword = openvz.vncpassword, - cpus = openvz.cpus, - ram = openvz.ram, - diskspace = openvz.diskspace, - rootpassword = openvz.rootpassword, - gateway4 = openvz.gateway4, - hostname = openvz.hostname, - gateway6 = openvz.gateway6, - template = resourceMapping[openvz.template], - templateId = resourceMapping[openvz.template].id + websocket_port=openvz.websocket_port, + websocket_pid=openvz.websocket_pid, + vncport=openvz.vncport, + vncpid=openvz.vncpid, + vncpassword=openvz.vncpassword, + cpus=openvz.cpus, + ram=openvz.ram, + diskspace=openvz.diskspace, + rootpassword=openvz.rootpassword, + gateway4=openvz.gateway4, + hostname=openvz.hostname, + gateway6=openvz.gateway6, + template=resourceMapping[openvz.template.id] ) newOpenVZ.save() - elementMapping[openvz] = newOpenVZ + elementMapping[openvz.id] = newOpenVZ from ...elements.openvz import OpenVZ_Interface as New_OpenVZ_Interface for openvz_interface in OpenVZ_Interface.objects.filter(): - newOpenVz_Interface = New_OpenVZ_Interface( - owner=userMapping[openvz_interface.owner], - ownerId=userMapping[openvz_interface.owner].id, + newOpenVZ_Interface = New_OpenVZ_Interface( + id="{0:0{1}x}".format(openvz_interface.id, 24), + owner=userMapping[openvz_interface.owner.id], state=openvz_interface.state, timeout=openvz_interface.timeout, name=openvz_interface.name, @@ -543,14 +197,14 @@ def migrate(): ipspy_pid=openvz_interface.ipspy_pid, used_addresses=openvz_interface.used_addresses ) - newOpenVz_Interface.save() - elementMapping[openvz_interface]=newOpenVz_Interface + newOpenVZ_Interface.save() + elementMapping[openvz_interface.id]=newOpenVZ_Interface - from ...elements.repy import Repy as RP + from ...elements.repy import Repy as NewRepy for repy in Repy.objects.filter(): - newRepy = RP( - owner=userMapping[repy.owner], - ownerId=userMapping[repy.owner].id, + newRepy = NewRepy( + id="{0:0{1}x}".format(repy.id, 24), + owner=userMapping[repy.owner.id], state=repy.state, timeout=repy.timeout, pid=repy.pid, @@ -563,17 +217,16 @@ def migrate(): cpus=repy.cpus, ram=repy.ram, bandwidth=repy.bandwidth, - template = resourceMapping[repy.template], - templateId = resourceMapping[repy.template].id + template=resourceMapping[repy.template.id] ) newRepy.save() - elementMapping[repy] = newRepy + elementMapping[repy.id] = newRepy from ...elements.repy import Repy_Interface as New_Repy_Interface for repy_interface in Repy_Interface.objects.filter(): newRepyInterface = New_Repy_Interface( - owner=userMapping[repy_interface.owner], - ownerId = userMapping[repy_interface.owner].id, + id="{0:0{1}x}".format(repy_interface.id, 24), + owner=userMapping[repy_interface.owner.id], state=repy_interface.state, timeout=repy_interface.timeout, name=repy_interface.name, @@ -581,14 +234,14 @@ def migrate(): used_addresses=repy_interface.used_addresses, ) newRepyInterface.save() - elementMapping[repy_interface] = newRepyInterface + elementMapping[repy_interface.id] = newRepyInterface from ...elements.tinc import Tinc as NewTc for tinc in Tinc.objects.filter(): newTinc = NewTc( - owner=userMapping[tinc.owner], - ownerId = userMapping[tinc.owner].id, + id="{0:0{1}x}".format(tinc.id, 24), + owner=userMapping[tinc.owner.id], state=tinc.state, timeout=tinc.timeout, port=tinc.port, @@ -599,14 +252,14 @@ def migrate(): peers=tinc.peers ) newTinc.save() - elementMapping[tinc] = newTinc + elementMapping[tinc.id] = newTinc from ...elements.udp_tunnel import UDP_Tunnel as NewUDP_Tunnel for udp_tunnel in UDP_Tunnel.objects.filter(): newUDP_Tunnel = NewUDP_Tunnel( - owner=userMapping[udp_tunnel.owner], - ownerId=userMapping[udp_tunnel.owner].id, + id="{0:0{1}x}".format(udp_tunnel.id, 24), + owner=userMapping[udp_tunnel.owner.id], state=udp_tunnel.state, timeout=udp_tunnel.timeout, pid=udp_tunnel.pid, @@ -614,80 +267,82 @@ def migrate(): connect=udp_tunnel.connect ) newUDP_Tunnel.save() - elementMapping[udp_tunnel] = newUDP_Tunnel + elementMapping[udp_tunnel.id] = newUDP_Tunnel from ...elements.vpncloud import VpnCloud as NewVPNCloud for vpncloud in VpnCloud.objects.filter(): newVPNcloud = NewVPNCloud( - owner=userMapping[vpncloud.owner], - ownerId=userMapping[vpncloud.owner].id, + id="{0:0{1}x}".format(vpncloud.id, 24), + owner=userMapping[vpncloud.owner.id], state=vpncloud.state, timeout=vpncloud.timeout, port=vpncloud.port, pid=vpncloud.pid, - network_id=vpncloud.network_id, + networkid=vpncloud.network_id, peers=vpncloud.peers ) newVPNcloud.save() - elementMapping[vpncloud] = newVPNcloud + elementMapping[vpncloud.id] = newVPNcloud connectionMapping={} from ...connections.bridge import Bridge as NewBridge - for bridge in Bridge.objects.filte(): + for bridge in Bridge.objects.filter(): newBridge = NewBridge( - owner=userMapping[bridge.owner], - ownerId = userMapping[bridge.owner].id, - state = bridge.state, - bridge= bridge.bridge, - emulation = bridge.emulation, - bandwidth_to = bridge.bandwidth_to, - bandwidth_from = bridge.bandwidth_from, - lossratio_to = bridge.lossratio_to, - lossratio_from = bridge.lossratio_from, - duplicate_to = bridge.duplicate_to, - duplicate_from = bridge.duplicate_from, - corrupt_to = bridge.corrupt_to, - corrupt_from = bridge.corrupt_from, - delay_to = bridge.delay_to, - delay_from = bridge.delay_from, - jitter_to = bridge.jitter_to, - jitter_from = bridge.jitter_from, - distribution_to = bridge.distribution_to, - distribution_from = bridge.distribution_from, - capturing = bridge.capturing, - capture_filter = bridge.capture_filter, - capture_port = bridge.capture_port, - capture_mode = bridge.capture_mode, - capture_pid = bridge.capture_pid) + id="{0:0{1}x}".format(bridge.id, 24), + owner=userMapping[bridge.owner.id], + state=bridge.state, + bridge=bridge.bridge, + emulation=bridge.emulation, + bandwidth_to=bridge.bandwidth_to, + bandwidth_from=bridge.bandwidth_from, + lossratio_to=bridge.lossratio_to, + lossratio_from=bridge.lossratio_from, + duplicate_to=bridge.duplicate_to, + duplicate_from=bridge.duplicate_from, + corrupt_to=bridge.corrupt_to, + corrupt_from=bridge.corrupt_from, + delay_to=bridge.delay_to, + delay_from=bridge.delay_from, + jitter_to=bridge.jitter_to, + jitter_from=bridge.jitter_from, + distribution_to=bridge.distribution_to, + distribution_from=bridge.distribution_from, + capturing=bridge.capturing, + capture_filter=bridge.capture_filter, + capture_port=bridge.capture_port, + capture_mode=bridge.capture_mode, + capture_pid=bridge.capture_pid) newBridge.save() - connectionMapping[bridge] = newBridge + connectionMapping[bridge.id] = newBridge from ...connections.fixed_bridge import Fixed_Bridge as NewFixedBridge for fixedBridge in Fixed_Bridge.objects.filter(): newFixedBridge = NewFixedBridge( - owner=userMapping[fixedBridge.owner], - ownerId = userMapping[fixedBridge.owner].id, + id="{0:0{1}x}".format(fixedBridge.id, 24), + owner=userMapping[fixedBridge.owner.id], state = fixedBridge.state) newFixedBridge.save() - connectionMapping[fixedBridge]=newFixedBridge + connectionMapping[fixedBridge.id]=newFixedBridge usageStatisticsMapping={} from ...accounting import UsageStatistics as NewUsageStatistics for usageStatistic in UsageStatistics.objects.filter(): newUsageStatistic = NewUsageStatistics( + id="{0:0{1}x}".format(usageStatistic.id, 24), begin =usageStatistic.begin, attrs=usageStatistic.attrs ) newUsageStatistic.save() - usageStatisticsMapping[usageStatistic] = newUsageStatistic + usageStatisticsMapping[usageStatistic.id] = newUsageStatistic usageRecordMapping={} from ...accounting import UsageRecord as NewUsageRecord for usageRecord in UsageRecord.objects.filter(): newUsageRecord = NewUsageRecord( - statistics=usageStatisticsMapping[usageRecord.statistics], + id="{0:0{1}x}".format(usageRecord.id, 24), + statistics=usageStatisticsMapping[usageRecord.statistics.id], type=usageRecord.type, begin=usageRecord.begin, end=usageRecord.end, @@ -698,171 +353,191 @@ def migrate(): cputime=usageRecord.cputime ) newUsageRecord.save() - usageRecordMapping[usageRecord]=newUsageRecord + usageRecordMapping[usageRecord.id]=newUsageRecord resourceInstanceMapping={} from ...resources import ResourceInstance as NewResourceInstance + for resourceInstance in ResourceInstance.objects.filter(): + ownerElement = None + if resourceInstance.ownerElement: + ownerElement=elementMapping[resourceInstance.ownerElement.id] + ownerConnection = None + if resourceInstance.ownerConnection: + ownerConnection=connectionMapping[resourceInstance.ownerConnection.id] newResourceInstance = NewResourceInstance( + id="{0:0{1}x}".format(resourceInstance.id, 24), type=resourceInstance.type, num=resourceInstance.num, - ownerElement=elementMapping(resourceInstance.ownerElement), - ownerElementId=elementMapping(resourceInstance.ownerElement).id, - ownerConnection=connectionMapping(resourceInstance.ownerConnection), - ownerConnectionId=connectionMapping(resourceInstance.ownerConnection).id, + ownerElement=ownerElement, + ownerConnection=ownerConnection, attrs=resourceInstance.attrs ) - + newResourceInstance.save() + resourceInstanceMapping[resourceInstance.id] = newResourceInstance #Update all elements to integrate the old references and relationships between them. for externalNetwork in External_Network.objects.filter(): - newExternalNetwork= elementMapping[externalNetwork] - newExternalNetwork.parent = elementMapping[externalNetwork.parent] - newExternalNetwork.parentId = elementMapping[externalNetwork.parent].id - newExternalNetwork.connection = connectionMapping[externalNetwork.connection] - newExternalNetwork.connectionId = connectionMapping[externalNetwork.connection].id - newExternalNetwork.usageStatistics = usageStatisticsMapping[externalNetwork.usageStatistic] - newExternalNetwork.usageStatisticsId = usageStatisticsMapping[externalNetwork.usageStatistic].id + newExternalNetwork= elementMapping.get(externalNetwork.id) + newExternalNetwork.parent = elementMapping.get(externalNetwork.parent.id if externalNetwork.parent else None) + newExternalNetwork.connection = connectionMapping.get(externalNetwork.connection.id if externalNetwork.connection else None) + newExternalNetwork.usageStatistics = usageStatisticsMapping.get(externalNetwork.usageStatistics.id if externalNetwork.usageStatistics else None) newExternalNetwork.save() for kvmqm in KVMQM.objects.filter(): - newKVMQM = elementMapping[kvmqm] - newKVMQM.parent = elementMapping[kvmqm.parent] - newKVMQM.parentId = elementMapping[kvmqm.parent].id - newKVMQM.connection = connectionMapping[kvmqm.connection] - newKVMQM.connectionId = connectionMapping[kvmqm.connection].id - newKVMQM.usageStatistics = usageStatisticsMapping[kvmqm.usageStatistic] - newKVMQM.usageStatisticsId = usageStatisticsMapping[kvmqm.usageStatistic].id + newKVMQM = elementMapping.get(kvmqm.id) + newKVMQM.parent = elementMapping.get(kvmqm.parent.id if kvmqm.parent else None) + newKVMQM.connection = connectionMapping.get(kvmqm.connection.id if kvmqm.connection else None) + newKVMQM.usageStatistics = usageStatisticsMapping.get(kvmqm.usageStatistics.id if kvmqm.usageStatistics else None) newKVMQM.save() - for kvmqm_interface in KVMQM_Interface.objects.filter(): - newKVMQM_Interface = elementMapping[kvmqm_interface] - newKVMQM_Interface.parent = elementMapping[kvmqm_interface.parent] - newKVMQM_Interface.parentId = elementMapping[kvmqm_interface.parent].id - newKVMQM_Interface.connection = connectionMapping[kvmqm_interface.connection] - newKVMQM_Interface.connectionId = connectionMapping[kvmqm_interface.connection].id - newKVMQM_Interface.usageStatistics = usageStatisticsMapping[kvmqm_interface.usageStatistic] - newKVMQM_Interface.usageStatisticsId = usageStatisticsMapping[kvmqm_interface.usageStatistic].id + for kvmqm_interface in KVM_Interface.objects.filter(): + newKVMQM_Interface = elementMapping.get(kvmqm_interface.id) + newKVMQM_Interface.parent = elementMapping.get(kvmqm_interface.parent.id if kvmqm_interface.parent else None) + newKVMQM_Interface.connection = connectionMapping.get(kvmqm_interface.connection.id if kvmqm_interface.connection else None) + newKVMQM_Interface.usageStatistics = usageStatisticsMapping.get(kvmqm_interface.usageStatistics.id if kvmqm_interface.usageStatistics else None) newKVMQM_Interface.save() - + """ for kvm in KVM.objects.filter(): - newKVM = elementMapping[kvm] - newKVM.parent = elementMapping[kvm.parent] - newKVM.parentId = elementMapping[kvm.parent].id - newKVM.connection = connectionMapping[kvm.connection] - newKVM.connectionId = connectionMapping[kvm.connection].id - newKVM.usageStatistics = usageStatisticsMapping[kvm.usageStatistic] - newKVM.usageStatisticsId = usageStatisticsMapping[kvm.usageStatistic].id + print "New KVM Object ????" + newKVM = elementMapping.get(kvm.id) + newKVM.parent = elementMapping.get(kvm.parent.id if kvm.parent else None) + newKVM.connection = connectionMapping.get(kvm.connection.id if kvm.connection else None) + newKVM.usageStatistics = usageStatisticsMapping.get(kvm.usageStatistics.id if kvm.usageStatistics else None) newKVM.save() for kvm_interface in KVM_Interface.objects.filter(): - newKVM_Interface = elementMapping[kvm_interface] - newKVM_Interface.parent = elementMapping[kvm_interface.parent] - newKVM_Interface.parentId = elementMapping[kvm_interface.parent].id - newKVM_Interface.connection = connectionMapping[kvm_interface.connection] - newKVM_Interface.connectionId = connectionMapping[kvm_interface.connection].id - newKVM_Interface.usageStatistics = usageStatisticsMapping[kvm_interface.usageStatistic] - newKVM_Interface.usageStatisticsId = usageStatisticsMapping[kvm_interface.usageStatistic].id + newKVM_Interface = elementMapping.get(kvm_interface.id) + print "[Old]Kvm_interface.parent.id=%s" % str(kvm_interface.parent.id) + print "[New]Kvm_interface.parent.id=%s" % str(elementMapping.get(kvm_interface.parent.id if kvm_interface.parent else None)) + if not elementMapping.get(kvm_interface.parent.id if kvm_interface.parent else None): + print "No daddy!" + newKVM_Interface.parent = elementMapping.get(kvm_interface.parent.id if kvm_interface.parent else None) + newKVM_Interface.connection = connectionMapping.get(kvm_interface.connection.id if kvm_interface.connection else None) + newKVM_Interface.usageStatistics = usageStatisticsMapping.get(kvm_interface.usageStatistics.id if kvm_interface.usageStatistics else None) newKVM_Interface.save() + """ for openvz in OpenVZ.objects.filter(): - newOpenVZ = elementMapping[openvz] - newOpenVZ.parent = elementMapping[openvz.parent] - newOpenVZ.parentId = elementMapping[openvz.parent].id - newOpenVZ.connection = connectionMapping[openvz.connection] - newOpenVZ.connectionId = connectionMapping[openvz.connection].id - newOpenVZ.usageStatistics = usageStatisticsMapping[openvz.usageStatistic] - newOpenVZ.usageStatisticsId = usageStatisticsMapping[openvz.usageStatistic].id + newOpenVZ = elementMapping.get(openvz.id) + newOpenVZ.parent = elementMapping.get(openvz.parent.id if openvz.parent else None) + newOpenVZ.connection = connectionMapping.get(openvz.connection.id if openvz.connection else None) + newOpenVZ.usageStatistics = usageStatisticsMapping.get(openvz.usageStatistics.id if openvz.usageStatistics else None) newOpenVZ.save() for openvz_interface in OpenVZ_Interface.objects.filter(): - newOpenVZ_Interface = elementMapping[openvz_interface] - newOpenVZ_Interface.parent = elementMapping[openvz_interface.parent] - newOpenVZ_Interface.parentId = elementMapping[openvz_interface.parent].id - newOpenVZ_Interface.connection = connectionMapping[openvz_interface.connection] - newOpenVZ_Interface.connectionId = connectionMapping[openvz_interface.connection].id - newOpenVZ_Interface.usageStatistics = usageStatisticsMapping[openvz_interface.usageStatistic] - newOpenVZ_Interface.usageStatisticsId = usageStatisticsMapping[openvz_interface.usageStatistic].id - newOpenVz_Interface.save() + newOpenVZ_Interface = elementMapping.get(openvz_interface.id) + newOpenVZ_Interface.parent = elementMapping.get(openvz_interface.parent.id if openvz_interface.parent else None) + newOpenVZ_Interface.connection = connectionMapping.get(openvz_interface.connection.id if openvz_interface.connection else None) + newOpenVZ_Interface.usageStatistics = usageStatisticsMapping.get(openvz_interface.usageStatistics.id if openvz_interface.usageStatistics else None) + newOpenVZ_Interface.save() for repy in Repy.objects.filter(): - newRepy = elementMapping[repy] - newRepy.parent = elementMapping[repy.parent] - newRepy.parentId = elementMapping[repy.parent].id - newRepy.connection = connectionMapping[repy.connection] - newRepy.connectionId = connectionMapping[repy.connection].id - newRepy.usageStatistics = usageStatisticsMapping[repy.usageStatistic] - newRepy.usageStatisticsId = usageStatisticsMapping[repy.usageStatistic].id + newRepy = elementMapping.get(repy.id) + newRepy.parent = elementMapping.get(repy.parent.id if repy.parent else None) + newRepy.connection = connectionMapping.get(repy.connection.id if repy.connection else None) + newRepy.usageStatistics = usageStatisticsMapping.get(repy.usageStatistics.id if repy.usageStatistics else None) newRepy.save() for repy_interface in Repy_Interface.objects.filter(): - newRepy_Interface = elementMapping[repy_interface] - newRepy_Interface.parent = elementMapping[repy_interface.parent] - newRepy_Interface.parentId = elementMapping[repy_interface.parent].id - newRepy_Interface.connection = connectionMapping[repy_interface.connection] - newRepy_Interface.connectionId = connectionMapping[repy_interface.connection].id - newRepy_Interface.usageStatistics = usageStatisticsMapping[repy_interface.usageStatistic] - newRepy_Interface.usageStatisticsId = usageStatisticsMapping[repy_interface.usageStatistic].id + newRepy_Interface = elementMapping.get(repy_interface.id) + newRepy_Interface.parent = elementMapping.get(repy_interface.parent.id if repy_interface.parent else None) + newRepy_Interface.connection = connectionMapping.get(repy_interface.connection.id if repy_interface.connection else None) + newRepy_Interface.usageStatistics = usageStatisticsMapping.get(repy_interface.usageStatistics.id if repy_interface.usageStatistics else None) newRepy_Interface.save() for tinc in Tinc.objects.filter(): - newTinc = elementMapping[tinc] - newTinc.parent = elementMapping[tinc.parent] - newTinc.parentId = elementMapping[tinc.parent].id - newTinc.connection = connectionMapping[tinc.connection] - newTinc.connectionId = connectionMapping[tinc.connection].id - newTinc.usageStatistics = usageStatisticsMapping[tinc.usageStatistic] - newTinc.usageStatisticsId = usageStatisticsMapping[tinc.usageStatistic].id + newTinc = elementMapping.get(tinc.id) + newTinc.parent = elementMapping.get(tinc.parent.id if tinc.parent else None) + newTinc.connection = connectionMapping.get(tinc.connection.id if tinc.connection else None) + newTinc.usageStatistics = usageStatisticsMapping.get(tinc.usageStatistics.id if tinc.usageStatistics else None) newTinc.save() for udp_tunnel in UDP_Tunnel.objects.filter(): - newUDP_Tunnel = elementMapping[udp_tunnel] - newUDP_Tunnel.parent = elementMapping[udp_tunnel.parent] - newUDP_Tunnel.parentId = elementMapping[udp_tunnel.parent].id - newUDP_Tunnel.connection = connectionMapping[udp_tunnel.connection] - newUDP_Tunnel.connectionId = connectionMapping[udp_tunnel.connection].id - newUDP_Tunnel.usageStatistics = usageStatisticsMapping[udp_tunnel.usageStatistic] - newUDP_Tunnel.usageStatisticsId = usageStatisticsMapping[udp_tunnel.usageStatistic].id + newUDP_Tunnel = elementMapping.get(udp_tunnel.id) + newUDP_Tunnel.parent = elementMapping.get(udp_tunnel.parent.id if udp_tunnel.parent else None) + newUDP_Tunnel.connection = connectionMapping.get(udp_tunnel.connection.id if udp_tunnel.connection else None) + newUDP_Tunnel.usageStatistics = usageStatisticsMapping.get(udp_tunnel.usageStatistics.id if udp_tunnel.usageStatistics else None) newUDP_Tunnel.save() for vpncloud in VpnCloud.objects.filter(): - newVpnCloud = elementMapping[vpncloud] - newVpnCloud.parent = elementMapping[vpncloud.parent] - newVpnCloud.parentId = elementMapping[vpncloud.parent].id - newVpnCloud.connection = connectionMapping[vpncloud.connection] - newVpnCloud.connectionId = connectionMapping[vpncloud.connection].id - newVpnCloud.usageStatistics = usageStatisticsMapping[vpncloud.usageStatistic] - newVpnCloud.usageStatisticsId = usageStatisticsMapping[vpncloud.usageStatistic].id + newVpnCloud = elementMapping.get(vpncloud.id) + newVpnCloud.parent = elementMapping.get(vpncloud.parent.id if vpncloud.parent else None) + newVpnCloud.connection = connectionMapping.get(vpncloud.connection.id if vpncloud.connection else None) + newVpnCloud.usageStatistics = usageStatisticsMapping.get(vpncloud.usageStatistics.id if vpncloud.usageStatistics else None) newVpnCloud.save() for bridge in Bridge.objects.filter(): - newBridge = connectionMapping[bridge] - newBridge.usageStatistics = usageStatisticsMapping[bridge.usageStatistic] - newBridge.usageStatisticsId = usageStatisticsMapping[bridge.usageStatistic].id + newBridge = connectionMapping.get(bridge.id) + newBridge.usageStatistics = usageStatisticsMapping.get(bridge.usageStatistics.id if bridge.usageStatistics else None) eleList=[] - for element in bridge.elements: - eleList.append(elementMapping[element]) + for element in bridge.elements.all(): + eleList.append(elementMapping.get(element.id)) newBridge.elements = eleList newBridge.save() for fixed_bridge in Fixed_Bridge.objects.filter(): - newFixedBridge = connectionMapping[fixed_bridge] - newFixedBridge.usageStatistics = usageStatisticsMapping[fixed_bridge.usageStatistic] - newFixedBridge.usageStatisticsId = usageStatisticsMapping[fixed_bridge.usageStatistic].id + newFixedBridge = connectionMapping.get(fixed_bridge.id) + newFixedBridge.usageStatistics = usageStatisticsMapping.get(fixed_bridge.usageStatistics.id if fixed_bridge.usageStatistics else None) eleList = [] - for element in fixed_bridge.elements: - eleList.append(elementMapping[element]) + for element in fixed_bridge.elements.all(): + eleList.append(elementMapping.get(element.id)) newFixedBridge.elements = eleList newFixedBridge.save() - - #TODO: Add connection(s) migration - #Test everythin once - #Add correct referencing of new objects for mongodb \ No newline at end of file + #Move all old element files to new element + """ + try: + kvmqm_interface_path = os.path.join(config.DATA_DIR, "kvmqm_interface") + if not os.path.exists(kvmqm_interface_path): + cmd_ = ["mkdir", kvmqm_interface_path] + out = cmd.run(cmd_) + except: + print "Couldn't create kvmqm_interface folder" + for element in Element.objects.filter(): + try: + oldPath = os.path.join(config.DATA_DIR, element.type, str(element.id)) + if element.type == "kvm_interface": + element.type = "kvmqm_interface" + if os.path.exists(oldPath): + newPath = os.path.join(config.DATA_DIR, element.type, str(elementMapping[element.id].id)) + cmd_ = ["mv", oldPath, newPath] + out = cmd.run(cmd_) + except Exception, e: + oldPath = os.path.join(config.DATA_DIR, element.type, str(element.id)) + if element.type == "kvm_interface": + element.type = "kvmqm_interface" + newPath = os.path.join(config.DATA_DIR, element.type, str(elementMapping[element.id].id)) + print "Couldn't move element folder: old_id=%s, new_id=%s, type=%s, path=%s, new_path=%s" \ + % (str(element.id), + str(elementMapping[element.id].id), + str(element.type), + str(oldPath), + str(newPath)) + print e + + for connection in Connection.objects.filter(): + try: + oldPath = os.path.join(config.DATA_DIR, connection.type, str(connection.id)) + if os.path.exists(oldPath): + newPath = os.path.join(config.DATA_DIR, connection.type, str(connectionMapping[connection.id].id)) + cmd_ = ["mv", oldPath, newPath] + out = cmd.run(cmd_) + except Exception, e: + oldPath = os.path.join(config.DATA_DIR, connection.type, str(connection.id)) + newPath = os.path.join(config.DATA_DIR, connection.type, str(connectionMapping[connection.id].id)) + print "Couldn't move connection folder: old_id=%s, new_id=%s, type=%s, path=%s, new_path=%s" \ + % (str(connection.id), + str(connectionMapping[connection.id].id), + str(connection.type), + str(oldPath), + str(newPath)) + raise e + """ diff --git a/hostmanager/tomato/elements/__init__.py b/hostmanager/tomato/elements/__init__.py index 13ed864d5..25d2cb751 100644 --- a/hostmanager/tomato/elements/__init__.py +++ b/hostmanager/tomato/elements/__init__.py @@ -141,7 +141,7 @@ def dataPath(self, filename=""): @param filename: a filename relative to the data path @type filename: str """ - return os.path.join(config.DATA_DIR, self.TYPE, str(self.id), filename) + return os.path.join(config.DATA_DIR, self.TYPE, str(self.getId()), filename) def upcast(self): """ @@ -163,6 +163,20 @@ class that it possesses. Due to a limitation of the database backend, def hasParent(self): return not self.parent is None + def getId(self): + """ + Returns the element id, converts automatically back to old postgres IDs if possible + :return: String + """ + id_ = str(self.id) + try: + #As we converted old ids by adding zeros to the left, we can check if the first 12 characters are zeros + if id_[0:12] == "000000000000": + id_ = int(id_, 16) + except: + pass + return id_ + def checkModify(self, attrs): """ Checks whether the attribute change can succeed before changing the @@ -331,7 +345,7 @@ def cap_attrs(cls): def info(self): res = { - "id": str(self.id), + "id": str(self.getId()), "type": self.type, "parent": str(self.parent.id) if self.hasParent() else None, "state": self.state, diff --git a/hostmanager/tomato/elements/kvm.py b/hostmanager/tomato/elements/kvm.py index 8f18b59cd..6e911828b 100644 --- a/hostmanager/tomato/elements/kvm.py +++ b/hostmanager/tomato/elements/kvm.py @@ -527,7 +527,6 @@ def _nlxtp_create_device_and_mountpoint(self): # if device file or mount point class KVM_Interface(elements.Element):# num = IntField() - name = StringField() mac = StringField() ipspy_pid = IntField() used_addresses = ListField(default=[]) @@ -604,7 +603,6 @@ def updateUsage(self, usage, data): "num": Attribute(field=num, schema=schema.Int(), readOnly=True), "mac": Attribute(field=mac, description="Mac Address", schema=schema.String(), readOnly=True), "ipspy_id": Attribute(field=ipspy_pid, schema=schema.Int(), readOnly=True), - "name": Attribute(field=name, description="Name", schema=schema.String(regex="^eth[0-9]+$")), "used_addresses": Attribute(field=used_addresses, schema=schema.List(), default=[], readOnly=True), }) diff --git a/hostmanager/tomato/elements/kvmqm.py b/hostmanager/tomato/elements/kvmqm.py index 9de059785..be7a77181 100644 --- a/hostmanager/tomato/elements/kvmqm.py +++ b/hostmanager/tomato/elements/kvmqm.py @@ -526,7 +526,6 @@ class KVMQM_Interface(elements.Element): num = IntField() - name = StringField() mac = StringField() ipspy_pid = IntField() used_addresses = ListField(default=[]) @@ -600,7 +599,6 @@ def updateUsage(self, usage, data): "num": Attribute(field=num, schema=schema.Int(), readOnly=True), "mac": Attribute(field=mac, description="Mac Address", schema=schema.String(), readOnly=True), "ipspy_id": Attribute(field=ipspy_pid, schema=schema.Int(), readOnly=True), - "name": Attribute(field=name, description="Name", schema=schema.String(regex="^eth[0-9]+$")), "used_addresses": Attribute(field=used_addresses, schema=schema.List(), default=[], readOnly=True), }) diff --git a/hostmanager/tomato/resources/__init__.py b/hostmanager/tomato/resources/__init__.py index ae8e7cc2b..9b3cdfc3e 100644 --- a/hostmanager/tomato/resources/__init__.py +++ b/hostmanager/tomato/resources/__init__.py @@ -96,7 +96,22 @@ def modify(self, attrs): self.attrs[key] = value logging.logMessage("info", category="resource", type=self.type, id=str(self.id), info=self.info()) self.update_or_save() - + + def getId(self): + """ + Returns the element id, converts automatically back to old postgres IDs if possible + :return: String + """ + id_ = self.id + try: + #As we converted old ids by adding zeros to the left, we can check if the first 12 characters are zeros + if self.id[0:12] == "000000000000": + oldId = int(id_, 16) + id_ = "{0:0{1}x}".format(oldId, 24) + except: + pass + return id_ + def remove(self): logging.logMessage("info", category="resource", type=self.type, id=str(self.id), info=self.info()) logging.logMessage("remove", category="resource", type=self.type, id=str(self.id)) @@ -104,7 +119,7 @@ def remove(self): def info(self): return { - "id": str(self.id), + "id": str(self.getId()), "type": self.type, "attrs": self.attrs.copy(), } @@ -118,15 +133,6 @@ class ResourceInstance(BaseDocument): ownerConnectionId = ReferenceFieldId(ownerConnection) attrs = DictField() - ACTIONS = {} - ATTRIBUTES = { - "id": IdAttribute(), - "type": Attribute(field=type, schema=schema.String(options=TYPES)), - "num": Attribute(field=num, schema=schema.Int(minValue=0)), - "ownerElement": Attribute(field=ownerElementId, schema=schema.Identifier()), - "ownerConnection": Attribute(field=ownerConnectionId, schema=schema.Identifier()), - } - def init(self, type, num, owner, attrs=None): #@ReservedAssignment if not attrs: attrs = {} self.type = type @@ -141,6 +147,29 @@ def init(self, type, num, owner, attrs=None): #@ReservedAssignment self.attrs = attrs self.save() + def getId(self): + """ + Returns the element id, converts automatically back to old postgres IDs if possible + :return: String + """ + id_ = str(self.id) + try: + #As we converted old ids by adding zeros to the left, we can check if the first 12 characters are zeros + if id_[0:12] == "000000000000": + id_ = int(id_, 16) + except: + pass + return id_ + + ACTIONS = {} + ATTRIBUTES = { + "id": IdAttribute(), + "type": Attribute(field=type, schema=schema.String(options=TYPES)), + "num": Attribute(field=num, schema=schema.Int(minValue=0)), + "ownerElement": Attribute(field=ownerElementId, schema=schema.Identifier()), + "ownerConnection": Attribute(field=ownerConnectionId, schema=schema.Identifier()), + } + def get(id_, **kwargs): try: From 94632615b8597bf794c05138d1f3fe6d85645918 Mon Sep 17 00:00:00 2001 From: Sebastian Wuest Date: Tue, 20 Jun 2017 14:01:56 +0200 Subject: [PATCH 7/7] Migration bugfix continued. --- hostmanager/tomato/connections/bridge.py | 2 +- hostmanager/tomato/elements/kvm.py | 44 +++++++++++++----------- hostmanager/tomato/elements/kvmqm.py | 43 +++++++++++++---------- hostmanager/tomato/elements/openvz.py | 35 +++++++++++-------- hostmanager/tomato/elements/repy.py | 42 ++++++++++++++-------- 5 files changed, 98 insertions(+), 68 deletions(-) diff --git a/hostmanager/tomato/connections/bridge.py b/hostmanager/tomato/connections/bridge.py index 9682db539..c632cf36a 100644 --- a/hostmanager/tomato/connections/bridge.py +++ b/hostmanager/tomato/connections/bridge.py @@ -73,7 +73,7 @@ def type(self): def init(self, *args, **kwargs): self.state = StateName.CREATED connections.Connection.init(self, *args, **kwargs) #no id and no attrs before this line - self.bridge = "br%s" % (str(self.getId())[13:24] if type(self.getId()) == type(str()) else str(self.getId())) + self.bridge = "br%s" % (str(self.getId())[13:24] if isinstance(self.getId(),type(str())) else str(self.getId())) self.capture_port = self.getResource("port") self.update_or_save(bridge=self.bridge, capture_port=self.capture_port) diff --git a/hostmanager/tomato/elements/kvm.py b/hostmanager/tomato/elements/kvm.py index 6e911828b..596f1abf3 100644 --- a/hostmanager/tomato/elements/kvm.py +++ b/hostmanager/tomato/elements/kvm.py @@ -138,7 +138,7 @@ class KVM(elements.Element, elements.RexTFVElement): vncport = IntField() vncpid = IntField() vncpassword =StringField() - cpus = IntField(default=1) + cpus = FloatField(default=1) ram = IntField(default=256) kblang = StringField(default=None, choises=kblang_options) usbtablet = BooleanField(default=True) @@ -359,27 +359,31 @@ def _imagePath(self, file="disk.qcow2"): #@ReservedAssignment return os.path.join(self._imagePathDir(), file) def updateUsage(self, usage, data): - self._checkState() - if self.state == StateName.CREATED: - return - if self.state == StateName.STARTED: - memory = 0 - cputime = 0 - stats = self.vir.getStatistics(self.vmid) - memory += stats.memory_used - cputime += stats.cputime_total - if self.vncpid and proc.isAlive(self.vncpid): - stats = proc.getStatistics(self.vncpid) - memory += stats.memory_used - cputime += stats.cputime_total - if self.websocket_pid and proc.isAlive(self.websocket_pid): - stats = proc.getStatistics(self.websocket_pid) + try: + self._checkState() + if self.state == StateName.CREATED: + return + if self.state == StateName.STARTED: + memory = 0 + cputime = 0 + stats = self.vir.getStatistics(self.vmid) memory += stats.memory_used cputime += stats.cputime_total - usage.memory = memory - usage.updateContinuous("cputime", cputime, data) - usage.diskspace = io.getSize(self._imagePathDir()) - + if self.vncpid and proc.isAlive(self.vncpid): + stats = proc.getStatistics(self.vncpid) + memory += stats.memory_used + cputime += stats.cputime_total + if self.websocket_pid and proc.isAlive(self.websocket_pid): + stats = proc.getStatistics(self.websocket_pid) + memory += stats.memory_used + cputime += stats.cputime_total + usage.memory = memory + usage.updateContinuous("cputime", cputime, data) + usage.diskspace = io.getSize(self._imagePathDir()) + except InternalError.WRONG_DATA: + pass + except: + raise """ RexTFV """ diff --git a/hostmanager/tomato/elements/kvmqm.py b/hostmanager/tomato/elements/kvmqm.py index be7a77181..6be4456a8 100644 --- a/hostmanager/tomato/elements/kvmqm.py +++ b/hostmanager/tomato/elements/kvmqm.py @@ -438,26 +438,31 @@ def info(self): return info def updateUsage(self, usage, data): - self._checkState() - if self.state == StateName.CREATED: - return - if self.state == StateName.STARTED: - memory = 0 - cputime = 0 - stats = qm.getStatistics(self.vmid) - memory += stats.memory_used - cputime += stats.cputime_total - if self.vncpid and proc.isAlive(self.vncpid): - stats = proc.getStatistics(self.vncpid) - memory += stats.memory_used - cputime += stats.cputime_total - if self.websocket_pid and proc.isAlive(self.websocket_pid): - stats = proc.getStatistics(self.websocket_pid) + try: + self._checkState() + if self.state == StateName.CREATED: + return + if self.state == StateName.STARTED: + memory = 0 + cputime = 0 + stats = qm.getStatistics(self.vmid) memory += stats.memory_used cputime += stats.cputime_total - usage.memory = memory - usage.updateContinuous("cputime", cputime, data) - usage.diskspace = io.getSize(self._imagePathDir()) + if self.vncpid and proc.isAlive(self.vncpid): + stats = proc.getStatistics(self.vncpid) + memory += stats.memory_used + cputime += stats.cputime_total + if self.websocket_pid and proc.isAlive(self.websocket_pid): + stats = proc.getStatistics(self.websocket_pid) + memory += stats.memory_used + cputime += stats.cputime_total + usage.memory = memory + usage.updateContinuous("cputime", cputime, data) + usage.diskspace = io.getSize(self._imagePathDir()) + except InternalError.WRONG_DATA: + pass + except: + raise ATTRIBUTES = elements.Element.ATTRIBUTES.copy() ATTRIBUTES.update({ @@ -527,6 +532,7 @@ class KVMQM_Interface(elements.Element): num = IntField() mac = StringField() + name = StringField() ipspy_pid = IntField() used_addresses = ListField(default=[]) @@ -598,6 +604,7 @@ def updateUsage(self, usage, data): ATTRIBUTES.update({ "num": Attribute(field=num, schema=schema.Int(), readOnly=True), "mac": Attribute(field=mac, description="Mac Address", schema=schema.String(), readOnly=True), + "name": Attribute(field=name, description="Name", schema=schema.String(regex="^eth[0-9]+$")), "ipspy_id": Attribute(field=ipspy_pid, schema=schema.Int(), readOnly=True), "used_addresses": Attribute(field=used_addresses, schema=schema.List(), default=[], readOnly=True), }) diff --git a/hostmanager/tomato/elements/openvz.py b/hostmanager/tomato/elements/openvz.py index 717ffd7a9..12754eab7 100644 --- a/hostmanager/tomato/elements/openvz.py +++ b/hostmanager/tomato/elements/openvz.py @@ -155,7 +155,7 @@ class OpenVZ(elements.Element, elements.RexTFVElement): vncport = IntField() vncpid = IntField() vncpassword =StringField() - cpus = IntField(default=1) + cpus = FloatField(default=1) ram = IntField(default=256) diskspace = IntField(default=10240) rootpassword = StringField() @@ -189,7 +189,7 @@ def init(self, *args, **kwargs): self.vncport = self.getResource("port") self.websocket_port = self.getResource("port", config.WEBSOCKIFY_PORT_BLACKLIST) self.vncpassword = cmd.randomPassword() - self.update_or_save() + self.update_or_save(vmid=self.vmid, vncport=self.vncport, websocket_port=self.websocket_port, vncpassword=self.vncpassword) #template: None, default template def _imagePath(self): @@ -596,18 +596,23 @@ def _diskspace(self): path.diskspace(self._imagePath()) def updateUsage(self, usage, data): - self._checkState() - if self.state == StateName.CREATED: - return - cputime = self._cputime() - if cputime: - usage.updateContinuous("cputime", cputime, data) - memory = self._memory() - if memory: - usage.memory = memory - diskspace = self._diskspace() - if diskspace: - usage.diskspace = diskspace + try: + self._checkState() + if self.state == StateName.CREATED: + return + cputime = self._cputime() + if cputime: + usage.updateContinuous("cputime", cputime, data) + memory = self._memory() + if memory: + usage.memory = memory + diskspace = self._diskspace() + if diskspace: + usage.diskspace = diskspace + except InternalError.WRONG_DATA: + pass + except: + raise ATTRIBUTES = elements.Element.ATTRIBUTES.copy() @@ -619,7 +624,7 @@ def updateUsage(self, usage, data): "vncpid": Attribute(field=vncpid, readOnly=True, schema=schema.Int()), "vncpassword": Attribute(field=vncpassword, readOnly=True, schema=schema.String(null=True)), "hostname": Attribute(field=hostname, label="Hostname", set=modify_hostname, schema=schema.String(null=True)), - "cpus": Attribute(field=cpus, label="Number of CPUs", schema=schema.Number(minValue=1,maxValue=4), default=1), + "cpus": Attribute(field=cpus, label="Number of CPUs", schema=schema.Number(minValue=0.1, maxValue=4.0), default=1.0), "ram": Attribute(field=ram, label="RAM", schema=schema.Int(minValue=64, maxValue=8192), default=256), "diskspace": Attribute(field=diskspace, label="Disk space in MB", schema=schema.Int(minValue=512, maxValue=102400), default=10240), "rootpassword": Attribute(field=rootpassword, label="Root password", schema=schema.String(null=True)), diff --git a/hostmanager/tomato/elements/repy.py b/hostmanager/tomato/elements/repy.py index 03f8263e1..51f02d5e9 100644 --- a/hostmanager/tomato/elements/repy.py +++ b/hostmanager/tomato/elements/repy.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see -import os, sys, shutil +import os, sys, time, shutil from ..db import * from ..generic import * from .. import connections, elements, config @@ -127,7 +127,7 @@ class Repy(elements.Element): vncpassword =StringField() args = ListField() args_doc = StringField(default=None, required=False) - cpus = IntField(default=1) + cpus = FloatField(default=1) ram = IntField(default=256) bandwidth = IntField(min_value=1024, max_value=10000000000, default=1000000) template = ReferenceField(template.Template, null=True) @@ -150,8 +150,14 @@ def type(self): return self.TYPE def init(self, *args, **kwargs): + self.state = StateName.PREPARED - elements.Element.init(self, *args, **kwargs) #no id and no attrs before this line + self.timeout = time.time() + config.MAX_TIMEOUT + #Workaround. We need repy to have a folder and be saved befor we start setting attributes. + self.update_or_save() + if not os.path.exists(self.dataPath()): + os.makedirs(self.dataPath()) + elements.Element.init(self, *args, **kwargs) self.vncport = self.getResource("port") self.websocket_port = self.getResource("port", config.WEBSOCKIFY_PORT_BLACKLIST) self.vncpassword = cmd.randomPassword() @@ -324,15 +330,20 @@ def info(self): return info def updateUsage(self, usage, data): - self._checkState() - usage.diskspace = path.diskspace(self.dataPath()) - if self.state == StateName.STARTED: - usage.memory = process.memory(self.pid) - cputime = process.cputime(self.pid) - if self.vncpid: - usage.memory += process.memory(self.vncpid) - cputime += process.cputime(self.vncpid) - usage.updateContinuous("cputime", cputime, data) + try: + self._checkState() + usage.diskspace = path.diskspace(self.dataPath()) + if self.state == StateName.STARTED: + usage.memory = process.memory(self.pid) + cputime = process.cputime(self.pid) + if self.vncpid: + usage.memory += process.memory(self.vncpid) + cputime += process.cputime(self.vncpid) + usage.updateContinuous("cputime", cputime, data) + except InternalError.WRONG_DATA: + pass + except: + raise ATTRIBUTES = elements.Element.ATTRIBUTES.copy() ATTRIBUTES.update({ @@ -344,8 +355,8 @@ def updateUsage(self, usage, data): "vncpassword": Attribute(field=vncpassword, readOnly=True, schema=schema.String()), "args": Attribute(field=args, description="Arguments", set=modify_args, schema = schema.List(), default=[]), "args_doc": Attribute(field=args_doc, description="Arguments Documentation", readOnly=True), - "cpus": Attribute(field=cpus, description="Number of CPUs", set=modify_cpus, schema=schema.Number(minValue=1,maxValue=4), default=1), - "ram": Attribute(field=ram, description="RAM", set=modify_ram, schema=schema.Int(minValue=64, maxValue=8192), default=256), + "cpus": Attribute(field=cpus, description="Number of CPUs", set=modify_cpus, schema=schema.Number(minValue=0.011,maxValue=4.0), default=0.25), + "ram": Attribute(field=ram, description="RAM", set=modify_ram, schema=schema.Int(minValue=10, maxValue=8192), default=25), "bandwidth": Attribute(field=bandwidth, description="Bandwidth in bytes/s", set=modify_bandwidth, schema=schema.Int(minValue=1024, maxValue=10000000000), default=1000000), "template": Attribute(field=templateId, description="Template", set=modify_template, schema=schema.Identifier()), }) @@ -431,10 +442,13 @@ def type(self): return self.TYPE def init(self, *args, **kwargs): + print "Init a repy interface element" self.state = StateName.PREPARED + print "State is set, now activate Element.Init for repy interface" elements.Element.init(self, *args, **kwargs) #no id and no attrs before this line assert isinstance(self.getParent(), Repy) self.name = self.getParent()._nextIfaceName() + self.update_or_save(name=self.name) def interfaceName(self): return self.getParent()._interfaceName(self.name)