diff --git a/backend_core/tomato/host/element.py b/backend_core/tomato/host/element.py index bd8a81fd0..a3c074ed7 100644 --- a/backend_core/tomato/host/element.py +++ b/backend_core/tomato/host/element.py @@ -125,6 +125,7 @@ def synchronize(self): if (not self.topologyElement and not self.topologyConnection) or (self.host is None): self.remove() return + self.updateInfo() self.modify(timeout=time.time() + settings.get_host_connections_settings()['component-timeout']) except error.UserError, err: if err.code == error.UserError.ENTITY_DOES_NOT_EXIST: @@ -144,6 +145,6 @@ def synchronize(id_): try: HostElement.objects.get(id=id_).synchronize() except DoesNotExist: - pass # nothong to synchronize + pass # nothing to synchronize scheduler.scheduleMaintenance(min(3600, settings.get_host_connections_settings()['component-timeout']), list, synchronize) \ No newline at end of file diff --git a/hostmanager/tomato/accounting.py b/hostmanager/tomato/accounting.py index ece481213..90b6db847 100644 --- a/hostmanager/tomato/accounting.py +++ b/hostmanager/tomato/accounting.py @@ -104,8 +104,8 @@ class UsageStatistics(BaseDocument): ATTRIBUTES = { "id": IdAttribute(), "begin": Attribute(field=begin, schema=schema.Number()), - "element": Attribute(get=lambda self: self.element.id if self.element else None), - "connection": Attribute(get=lambda self: self.connection.id if self.element else None) + "element": Attribute(get=lambda self: self.element.getId() if self.element else None), + "connection": Attribute(get=lambda self: self.connection.getId() if self.connection else None) } meta = { diff --git a/hostmanager/tomato/api/accounting.py b/hostmanager/tomato/api/accounting.py index 1cf6b9d96..c71d54f8e 100644 --- a/hostmanager/tomato/api/accounting.py +++ b/hostmanager/tomato/api/accounting.py @@ -54,8 +54,8 @@ def accounting_statistics(type=None, after=None, before=None): #@ReservedAssignm are the connection ids (as strings) and the values are the usage statistics. """ - elSt = dict([(str(el.id), el.getUsageStatistics().info(type, after, before)) for el in elements.getAll(owner=currentUser())]) - conSt = dict([(str(con.id), con.getUsageStatistics().info(type, after, before)) for con in connections.getAll(owner=currentUser())]) + elSt = dict([(str(el.getId()), el.getUsageStatistics().info(type, after, before)) for el in elements.getAll(owner=currentUser())]) + conSt = dict([(str(con.getId()), con.getUsageStatistics().info(type, after, before)) for con in connections.getAll(owner=currentUser())]) return {"elements": elSt, "connections": conSt} def accounting_element_statistics(id, type=None, after=None, before=None): #@ReservedAssignment diff --git a/hostmanager/tomato/api/connections.py b/hostmanager/tomato/api/connections.py index 08cb968a8..1ff76d013 100644 --- a/hostmanager/tomato/api/connections.py +++ b/hostmanager/tomato/api/connections.py @@ -16,6 +16,14 @@ # along with this program. If not, see def _getConnection(id_): + import sys + try: + id_ = str(id_) + if sys.getsizeof(id_) < 60: + oldId=int(str(id_)) + id_ = "{0:0{1}x}".format(oldId, 24) + except: + pass con = connections.get(id_, owner=currentUser()) UserError.check(con, UserError.ENTITY_DOES_NOT_EXIST, "No such connection", data={"id": id_}) return con diff --git a/hostmanager/tomato/api/elements.py b/hostmanager/tomato/api/elements.py index 0a2764b82..b6a0b41bd 100644 --- a/hostmanager/tomato/api/elements.py +++ b/hostmanager/tomato/api/elements.py @@ -16,6 +16,14 @@ # along with this program. If not, see def _getElement(id_): + import sys + try: + id_ = str(id_) + if sys.getsizeof(id_) < 60: + oldId=int(str(id_)) + id_ = "{0:0{1}x}".format(oldId, 24) + except: + pass el = elements.get(id_, owner=currentUser()) UserError.check(el, UserError.ENTITY_DOES_NOT_EXIST, "No such element", data={"id": id_}) return el diff --git a/hostmanager/tomato/api/resources.py b/hostmanager/tomato/api/resources.py index ab0b293e7..8092bf2a9 100644 --- a/hostmanager/tomato/api/resources.py +++ b/hostmanager/tomato/api/resources.py @@ -16,6 +16,14 @@ # along with this program. If not, see def _getResource(id_): + import sys + try: + id_ = str(id_) + if sys.getsizeof(id_) < 60: + oldId=int(str(id_)) + id_ = "{0:0{1}x}".format(oldId, 24) + except: + pass res = resources.get(id_) UserError.check(res, UserError.ENTITY_DOES_NOT_EXIST, "No such resource", data={"id": id_}) return res diff --git a/hostmanager/tomato/connections/__init__.py b/hostmanager/tomato/connections/__init__.py index c34ba3868..b09763de5 100644 --- a/hostmanager/tomato/connections/__init__.py +++ b/hostmanager/tomato/connections/__init__.py @@ -156,6 +156,20 @@ class that it possesses. Due to a limitation of the database backend, pass raise InternalError(message="Failed to cast connection", code=InternalError.UPCAST, data={"id": str(self.id), "type": self.type}) + def getId(self): + """ + Returns the element id, converts automatically back to old postgres IDs if possible + :return: String + """ + id_ = str(self.id) + try: + #As we converted old ids by adding zeros to the left, we can check if the first 12 characters are zeros + if id_[0:12] == "000000000000": + id_ = int(id_, 16) + except: + pass + return id_ + def dataPath(self, filename=""): """ This method can be used to create filenames relative to a directory @@ -170,7 +184,7 @@ def dataPath(self, filename=""): @param filename: a filename relative to the data path @type filename: str """ - return os.path.join(config.DATA_DIR, self.TYPE, str(self.id), filename) + return os.path.join(config.DATA_DIR, self.TYPE, str(self.getId()), filename) @classmethod def determineConcept(cls, el1, el2): @@ -274,7 +288,7 @@ def getElements(self): def info(self): els = [str(el.id) for el in self.elements] return { - "id": str(self.id), + "id": str(self.getId()), "type": self.type, "state": self.state, "attrs": Entity.info(self), diff --git a/hostmanager/tomato/connections/bridge.py b/hostmanager/tomato/connections/bridge.py index 6bddee00e..c632cf36a 100644 --- a/hostmanager/tomato/connections/bridge.py +++ b/hostmanager/tomato/connections/bridge.py @@ -73,8 +73,9 @@ def type(self): def init(self, *args, **kwargs): self.state = StateName.CREATED connections.Connection.init(self, *args, **kwargs) #no id and no attrs before this line - self.bridge = "br%s" % str(self.id)[13:24] + self.bridge = "br%s" % (str(self.getId())[13:24] if isinstance(self.getId(),type(str())) else str(self.getId())) self.capture_port = self.getResource("port") + self.update_or_save(bridge=self.bridge, capture_port=self.capture_port) def _startCapturing(self): if not self.capturing or self.state == StateName.CREATED: @@ -245,6 +246,7 @@ def action_stop(self): if net.bridgeExists(self.bridge): net.ifDown(self.bridge) net.bridgeRemove(self.bridge) + self.update_or_save(bridge=self.bridge, capture_port=self.capture_port) self.setState(StateName.CREATED) def remove(self): diff --git a/hostmanager/tomato/db/migrations/migration_0001.py b/hostmanager/tomato/db/migrations/migration_0001.py index 45680ad91..c31932d89 100644 --- a/hostmanager/tomato/db/migrations/migration_0001.py +++ b/hostmanager/tomato/db/migrations/migration_0001.py @@ -1,3 +1,543 @@ +from ...lib.newcmd.util import run, CommandError, cmd, proc +from ... import config +import os +from ...modelsold import User, Resource, Template, Network, Element, External_Network, KVM, KVM_Interface, KVMQM, KVMQM_Interface, OpenVZ, OpenVZ_Interface, Repy, Repy_Interface, Tinc, UDP_Tunnel, VpnCloud, Connection, Bridge, Fixed_Bridge, UsageRecord, UsageStatistics, ResourceInstance + +''' +Migration through 2 Steps: +1. Create old elements in the new database with all attributes except for references +2. Update all elements to integrate the old references and relationships between them. +''' def migrate(): - pass \ No newline at end of file + + #Create old elements in the new database with all attributes except for references + from django.core.management import call_command + call_command('syncdb', verbosity=0) + + userMapping = {} + from ...user import User as NewUser + for user in User.objects.filter(): + newUser = NewUser( + id="{0:0{1}x}".format(user.id, 24), + name=user.name) + newUser.save() + userMapping[user.id]=newUser + + resourceMapping = {} + from ...resources import Resource as NewResource + for resource in Resource.objects.filter(): + if resource.type not in ["network","template"]: + newResource = NewResource( + id="{0:0{1}x}".format(resource.id, 24), + attrs=resource.attrs, + type=resource.type) + newResource.save() + resourceMapping[resource.id] = newResource + + from ...resources.template import Template as NewTemplate + for template in Template.objects.filter(): + newTemplate = NewTemplate( + id="{0:0{1}x}".format(template.id, 24), + attrs=resource.attrs, + type=resource.type, + owner=userMapping[template.owner.id], + tech=template.tech, + name=template.name, + preference=template.preference, + urls=template.urls, + checksum=template.checksum, + size=template.size, + popularity=template.popularity, + ready=template.ready, + kblang=template.kblang) + newTemplate.save() + resourceMapping[template.id] = newTemplate + + from ...resources.network import Network as NewNetwork + for network in Network.objects.filter(): + newNetwork = NewNetwork( + id="{0:0{1}x}".format(network.id, 24), + attrs=resource.attrs, + type=resource.type, + owner=userMapping[network.owner.id], + kind=network.kind, + bridge=network.bridge, + preference=network.preference) + resourceMapping[network.id] = newNetwork + + elementMapping = {} + from ...elements.external_network import External_Network as NewExternalNetwork + for externalNetwork in External_Network.objects.filter(): + newExternalNetwork = NewExternalNetwork( + id="{0:0{1}x}".format(externalNetwork.id, 24), + owner=userMapping[externalNetwork.owner.id], + state=externalNetwork.state, + timeout=externalNetwork.timeout, + network=resourceMapping[externalNetwork.network.id] + ) + elementMapping[externalNetwork.id] = newExternalNetwork + + + from ...elements.kvmqm import KVMQM as New_KVMQM + for kvmqm in KVMQM.objects.filter(): + newKVMQM = New_KVMQM( + id="{0:0{1}x}".format(kvmqm.id, 24), + owner=userMapping[kvmqm.owner.id], + state=kvmqm.state, + timeout=kvmqm.timeout, + vmid=kvmqm.vmid, + websocket_port=kvmqm.websocket_port, + websocket_pid=kvmqm.websocket_pid, + vncport=kvmqm.vncport, + vncpid=kvmqm.vncpid, + vncpassword=kvmqm.vncpassword, + cpus=kvmqm.cpus, + ram=kvmqm.ram, + kblang=kvmqm.kblang, + usbtablet=kvmqm.usbtablet, + template=resourceMapping[kvmqm.template.id] + ) + newKVMQM.save() + elementMapping[kvmqm.id] = newKVMQM + + from ...elements.kvmqm import KVMQM_Interface as New_KVMQM_Interface + for kvmqm_interface in KVM_Interface.objects.filter(): #This is due to an old naming error (Should have been KVMQM_Interface) + newKVMQMInterface = New_KVMQM_Interface( + id="{0:0{1}x}".format(kvmqm_interface.id, 24), + owner=userMapping[kvmqm_interface.owner.id], + state=kvmqm_interface.state, + timeout=kvmqm_interface.timeout, + num=kvmqm_interface.num, + mac=kvmqm_interface.mac, + ipspy_pid=kvmqm_interface.ipspy_pid, + used_addresses=kvmqm_interface.used_addresses + ) + newKVMQMInterface.save() + elementMapping[kvmqm_interface.id] = newKVMQMInterface + + + #They should never exist + """ + from ...elements.kvm import KVM as New_KVM + for kvm in KVM.objects.filter(): + newKVM = New_KVM( + id="{0:0{1}x}".format(kvm.id, 24), + owner=userMapping[kvm.owner.id], + state=kvm.state, + timeout=kvm.timeout, + vmid=kvm.vmid, + websocket_port=kvm.websocket_port, + websocket_pid=kvm.websocket_pid, + vncport=kvm.vncport, + vncpid=kvm.vncpid, + vncpassword=kvm.vncpassword, + cpus=kvm.cpus, + ram=kvm.ram, + kblang=kvm.kblang, + usbtablet=kvm.usbtablet, + template = resourceMapping[kvm.template.id] + ) + newKVM.save() + elementMapping[kvm.id] = newKVM + + from ...elements.kvm import KVM_Interface as New_KVM_Interface + for kvm_interface in KVM_Interface.objects.filter(): + newKVMInterface = New_KVM_Interface( + id="{0:0{1}x}".format(kvm_interface.id, 24), + owner=userMapping[kvm_interface.owner.id], + state=kvm_interface.state, + timeout=kvm_interface.timeout, + num=kvm_interface.num, + mac=kvm_interface.mac, + ipspy_pid=kvm_interface.ipspy_pid, + used_addresses=kvm_interface.used_addresses + ) + newKVMInterface.save() + elementMapping[kvm_interface.id] = newKVMInterface + """ + + from ...elements.openvz import OpenVZ as New_OpenVZ + for openvz in OpenVZ.objects.filter(): + newOpenVZ = New_OpenVZ( + id="{0:0{1}x}".format(openvz.id, 24), + owner=userMapping[openvz.owner.id], + state=openvz.state, + timeout=openvz.timeout, + vmid=openvz.vmid, + websocket_port=openvz.websocket_port, + websocket_pid=openvz.websocket_pid, + vncport=openvz.vncport, + vncpid=openvz.vncpid, + vncpassword=openvz.vncpassword, + cpus=openvz.cpus, + ram=openvz.ram, + diskspace=openvz.diskspace, + rootpassword=openvz.rootpassword, + gateway4=openvz.gateway4, + hostname=openvz.hostname, + gateway6=openvz.gateway6, + template=resourceMapping[openvz.template.id] + ) + newOpenVZ.save() + elementMapping[openvz.id] = newOpenVZ + + from ...elements.openvz import OpenVZ_Interface as New_OpenVZ_Interface + for openvz_interface in OpenVZ_Interface.objects.filter(): + newOpenVZ_Interface = New_OpenVZ_Interface( + id="{0:0{1}x}".format(openvz_interface.id, 24), + owner=userMapping[openvz_interface.owner.id], + state=openvz_interface.state, + timeout=openvz_interface.timeout, + name=openvz_interface.name, + ip4address=openvz_interface.ip4address, + ip6address=openvz_interface.ip6address, + use_dhcp=openvz_interface.use_dhcp, + mac=openvz_interface.mac, + ipspy_pid=openvz_interface.ipspy_pid, + used_addresses=openvz_interface.used_addresses + ) + newOpenVZ_Interface.save() + elementMapping[openvz_interface.id]=newOpenVZ_Interface + + from ...elements.repy import Repy as NewRepy + for repy in Repy.objects.filter(): + newRepy = NewRepy( + id="{0:0{1}x}".format(repy.id, 24), + owner=userMapping[repy.owner.id], + state=repy.state, + timeout=repy.timeout, + pid=repy.pid, + websocket_port=repy.websocket_port, + websocket_pid=repy.websocket_pid, + vncport=repy.vncport, + vncpid=repy.vncpid, + vncpassword=repy.vncpassword, + args=repy.args, + cpus=repy.cpus, + ram=repy.ram, + bandwidth=repy.bandwidth, + template=resourceMapping[repy.template.id] + ) + newRepy.save() + elementMapping[repy.id] = newRepy + + from ...elements.repy import Repy_Interface as New_Repy_Interface + for repy_interface in Repy_Interface.objects.filter(): + newRepyInterface = New_Repy_Interface( + id="{0:0{1}x}".format(repy_interface.id, 24), + owner=userMapping[repy_interface.owner.id], + state=repy_interface.state, + timeout=repy_interface.timeout, + name=repy_interface.name, + ipspy_pid=repy_interface.ipspy_pid, + used_addresses=repy_interface.used_addresses, + ) + newRepyInterface.save() + elementMapping[repy_interface.id] = newRepyInterface + + + from ...elements.tinc import Tinc as NewTc + for tinc in Tinc.objects.filter(): + newTinc = NewTc( + id="{0:0{1}x}".format(tinc.id, 24), + owner=userMapping[tinc.owner.id], + state=tinc.state, + timeout=tinc.timeout, + port=tinc.port, + path=tinc.path, + mode=tinc.mode, + privkey=tinc.privkey, + pubkey=tinc.pubkey, + peers=tinc.peers + ) + newTinc.save() + elementMapping[tinc.id] = newTinc + + + from ...elements.udp_tunnel import UDP_Tunnel as NewUDP_Tunnel + for udp_tunnel in UDP_Tunnel.objects.filter(): + newUDP_Tunnel = NewUDP_Tunnel( + id="{0:0{1}x}".format(udp_tunnel.id, 24), + owner=userMapping[udp_tunnel.owner.id], + state=udp_tunnel.state, + timeout=udp_tunnel.timeout, + pid=udp_tunnel.pid, + port=udp_tunnel.port, + connect=udp_tunnel.connect + ) + newUDP_Tunnel.save() + elementMapping[udp_tunnel.id] = newUDP_Tunnel + + + from ...elements.vpncloud import VpnCloud as NewVPNCloud + for vpncloud in VpnCloud.objects.filter(): + newVPNcloud = NewVPNCloud( + id="{0:0{1}x}".format(vpncloud.id, 24), + owner=userMapping[vpncloud.owner.id], + state=vpncloud.state, + timeout=vpncloud.timeout, + port=vpncloud.port, + pid=vpncloud.pid, + networkid=vpncloud.network_id, + peers=vpncloud.peers + ) + newVPNcloud.save() + elementMapping[vpncloud.id] = newVPNcloud + + + connectionMapping={} + from ...connections.bridge import Bridge as NewBridge + for bridge in Bridge.objects.filter(): + newBridge = NewBridge( + id="{0:0{1}x}".format(bridge.id, 24), + owner=userMapping[bridge.owner.id], + state=bridge.state, + bridge=bridge.bridge, + emulation=bridge.emulation, + bandwidth_to=bridge.bandwidth_to, + bandwidth_from=bridge.bandwidth_from, + lossratio_to=bridge.lossratio_to, + lossratio_from=bridge.lossratio_from, + duplicate_to=bridge.duplicate_to, + duplicate_from=bridge.duplicate_from, + corrupt_to=bridge.corrupt_to, + corrupt_from=bridge.corrupt_from, + delay_to=bridge.delay_to, + delay_from=bridge.delay_from, + jitter_to=bridge.jitter_to, + jitter_from=bridge.jitter_from, + distribution_to=bridge.distribution_to, + distribution_from=bridge.distribution_from, + capturing=bridge.capturing, + capture_filter=bridge.capture_filter, + capture_port=bridge.capture_port, + capture_mode=bridge.capture_mode, + capture_pid=bridge.capture_pid) + newBridge.save() + connectionMapping[bridge.id] = newBridge + + from ...connections.fixed_bridge import Fixed_Bridge as NewFixedBridge + for fixedBridge in Fixed_Bridge.objects.filter(): + newFixedBridge = NewFixedBridge( + id="{0:0{1}x}".format(fixedBridge.id, 24), + owner=userMapping[fixedBridge.owner.id], + state = fixedBridge.state) + newFixedBridge.save() + connectionMapping[fixedBridge.id]=newFixedBridge + + usageStatisticsMapping={} + from ...accounting import UsageStatistics as NewUsageStatistics + for usageStatistic in UsageStatistics.objects.filter(): + newUsageStatistic = NewUsageStatistics( + id="{0:0{1}x}".format(usageStatistic.id, 24), + begin =usageStatistic.begin, + attrs=usageStatistic.attrs + ) + newUsageStatistic.save() + usageStatisticsMapping[usageStatistic.id] = newUsageStatistic + + usageRecordMapping={} + from ...accounting import UsageRecord as NewUsageRecord + for usageRecord in UsageRecord.objects.filter(): + newUsageRecord = NewUsageRecord( + id="{0:0{1}x}".format(usageRecord.id, 24), + statistics=usageStatisticsMapping[usageRecord.statistics.id], + type=usageRecord.type, + begin=usageRecord.begin, + end=usageRecord.end, + measurements=usageRecord.measurements, + memory=usageRecord.memory, + diskspace=usageRecord.diskspace, + traffic=usageRecord.traffic, + cputime=usageRecord.cputime + ) + newUsageRecord.save() + usageRecordMapping[usageRecord.id]=newUsageRecord + + resourceInstanceMapping={} + from ...resources import ResourceInstance as NewResourceInstance + + for resourceInstance in ResourceInstance.objects.filter(): + ownerElement = None + if resourceInstance.ownerElement: + ownerElement=elementMapping[resourceInstance.ownerElement.id] + ownerConnection = None + if resourceInstance.ownerConnection: + ownerConnection=connectionMapping[resourceInstance.ownerConnection.id] + newResourceInstance = NewResourceInstance( + id="{0:0{1}x}".format(resourceInstance.id, 24), + type=resourceInstance.type, + num=resourceInstance.num, + ownerElement=ownerElement, + ownerConnection=ownerConnection, + attrs=resourceInstance.attrs + ) + newResourceInstance.save() + resourceInstanceMapping[resourceInstance.id] = newResourceInstance + + + #Update all elements to integrate the old references and relationships between them. + + for externalNetwork in External_Network.objects.filter(): + newExternalNetwork= elementMapping.get(externalNetwork.id) + newExternalNetwork.parent = elementMapping.get(externalNetwork.parent.id if externalNetwork.parent else None) + newExternalNetwork.connection = connectionMapping.get(externalNetwork.connection.id if externalNetwork.connection else None) + newExternalNetwork.usageStatistics = usageStatisticsMapping.get(externalNetwork.usageStatistics.id if externalNetwork.usageStatistics else None) + newExternalNetwork.save() + + for kvmqm in KVMQM.objects.filter(): + newKVMQM = elementMapping.get(kvmqm.id) + newKVMQM.parent = elementMapping.get(kvmqm.parent.id if kvmqm.parent else None) + newKVMQM.connection = connectionMapping.get(kvmqm.connection.id if kvmqm.connection else None) + newKVMQM.usageStatistics = usageStatisticsMapping.get(kvmqm.usageStatistics.id if kvmqm.usageStatistics else None) + newKVMQM.save() + + for kvmqm_interface in KVM_Interface.objects.filter(): + newKVMQM_Interface = elementMapping.get(kvmqm_interface.id) + newKVMQM_Interface.parent = elementMapping.get(kvmqm_interface.parent.id if kvmqm_interface.parent else None) + newKVMQM_Interface.connection = connectionMapping.get(kvmqm_interface.connection.id if kvmqm_interface.connection else None) + newKVMQM_Interface.usageStatistics = usageStatisticsMapping.get(kvmqm_interface.usageStatistics.id if kvmqm_interface.usageStatistics else None) + newKVMQM_Interface.save() + + """ + for kvm in KVM.objects.filter(): + print "New KVM Object ????" + newKVM = elementMapping.get(kvm.id) + newKVM.parent = elementMapping.get(kvm.parent.id if kvm.parent else None) + newKVM.connection = connectionMapping.get(kvm.connection.id if kvm.connection else None) + newKVM.usageStatistics = usageStatisticsMapping.get(kvm.usageStatistics.id if kvm.usageStatistics else None) + newKVM.save() + + for kvm_interface in KVM_Interface.objects.filter(): + newKVM_Interface = elementMapping.get(kvm_interface.id) + print "[Old]Kvm_interface.parent.id=%s" % str(kvm_interface.parent.id) + print "[New]Kvm_interface.parent.id=%s" % str(elementMapping.get(kvm_interface.parent.id if kvm_interface.parent else None)) + if not elementMapping.get(kvm_interface.parent.id if kvm_interface.parent else None): + print "No daddy!" + newKVM_Interface.parent = elementMapping.get(kvm_interface.parent.id if kvm_interface.parent else None) + newKVM_Interface.connection = connectionMapping.get(kvm_interface.connection.id if kvm_interface.connection else None) + newKVM_Interface.usageStatistics = usageStatisticsMapping.get(kvm_interface.usageStatistics.id if kvm_interface.usageStatistics else None) + newKVM_Interface.save() + """ + + for openvz in OpenVZ.objects.filter(): + newOpenVZ = elementMapping.get(openvz.id) + newOpenVZ.parent = elementMapping.get(openvz.parent.id if openvz.parent else None) + newOpenVZ.connection = connectionMapping.get(openvz.connection.id if openvz.connection else None) + newOpenVZ.usageStatistics = usageStatisticsMapping.get(openvz.usageStatistics.id if openvz.usageStatistics else None) + newOpenVZ.save() + + for openvz_interface in OpenVZ_Interface.objects.filter(): + newOpenVZ_Interface = elementMapping.get(openvz_interface.id) + newOpenVZ_Interface.parent = elementMapping.get(openvz_interface.parent.id if openvz_interface.parent else None) + newOpenVZ_Interface.connection = connectionMapping.get(openvz_interface.connection.id if openvz_interface.connection else None) + newOpenVZ_Interface.usageStatistics = usageStatisticsMapping.get(openvz_interface.usageStatistics.id if openvz_interface.usageStatistics else None) + newOpenVZ_Interface.save() + + + for repy in Repy.objects.filter(): + newRepy = elementMapping.get(repy.id) + newRepy.parent = elementMapping.get(repy.parent.id if repy.parent else None) + newRepy.connection = connectionMapping.get(repy.connection.id if repy.connection else None) + newRepy.usageStatistics = usageStatisticsMapping.get(repy.usageStatistics.id if repy.usageStatistics else None) + newRepy.save() + + for repy_interface in Repy_Interface.objects.filter(): + newRepy_Interface = elementMapping.get(repy_interface.id) + newRepy_Interface.parent = elementMapping.get(repy_interface.parent.id if repy_interface.parent else None) + newRepy_Interface.connection = connectionMapping.get(repy_interface.connection.id if repy_interface.connection else None) + newRepy_Interface.usageStatistics = usageStatisticsMapping.get(repy_interface.usageStatistics.id if repy_interface.usageStatistics else None) + newRepy_Interface.save() + + for tinc in Tinc.objects.filter(): + newTinc = elementMapping.get(tinc.id) + newTinc.parent = elementMapping.get(tinc.parent.id if tinc.parent else None) + newTinc.connection = connectionMapping.get(tinc.connection.id if tinc.connection else None) + newTinc.usageStatistics = usageStatisticsMapping.get(tinc.usageStatistics.id if tinc.usageStatistics else None) + newTinc.save() + + for udp_tunnel in UDP_Tunnel.objects.filter(): + newUDP_Tunnel = elementMapping.get(udp_tunnel.id) + newUDP_Tunnel.parent = elementMapping.get(udp_tunnel.parent.id if udp_tunnel.parent else None) + newUDP_Tunnel.connection = connectionMapping.get(udp_tunnel.connection.id if udp_tunnel.connection else None) + newUDP_Tunnel.usageStatistics = usageStatisticsMapping.get(udp_tunnel.usageStatistics.id if udp_tunnel.usageStatistics else None) + newUDP_Tunnel.save() + + for vpncloud in VpnCloud.objects.filter(): + newVpnCloud = elementMapping.get(vpncloud.id) + newVpnCloud.parent = elementMapping.get(vpncloud.parent.id if vpncloud.parent else None) + newVpnCloud.connection = connectionMapping.get(vpncloud.connection.id if vpncloud.connection else None) + newVpnCloud.usageStatistics = usageStatisticsMapping.get(vpncloud.usageStatistics.id if vpncloud.usageStatistics else None) + newVpnCloud.save() + + for bridge in Bridge.objects.filter(): + newBridge = connectionMapping.get(bridge.id) + newBridge.usageStatistics = usageStatisticsMapping.get(bridge.usageStatistics.id if bridge.usageStatistics else None) + + eleList=[] + for element in bridge.elements.all(): + eleList.append(elementMapping.get(element.id)) + newBridge.elements = eleList + newBridge.save() + + for fixed_bridge in Fixed_Bridge.objects.filter(): + newFixedBridge = connectionMapping.get(fixed_bridge.id) + newFixedBridge.usageStatistics = usageStatisticsMapping.get(fixed_bridge.usageStatistics.id if fixed_bridge.usageStatistics else None) + + eleList = [] + for element in fixed_bridge.elements.all(): + eleList.append(elementMapping.get(element.id)) + newFixedBridge.elements = eleList + newFixedBridge.save() + + + #Move all old element files to new element + """ + try: + kvmqm_interface_path = os.path.join(config.DATA_DIR, "kvmqm_interface") + if not os.path.exists(kvmqm_interface_path): + cmd_ = ["mkdir", kvmqm_interface_path] + out = cmd.run(cmd_) + except: + print "Couldn't create kvmqm_interface folder" + for element in Element.objects.filter(): + try: + oldPath = os.path.join(config.DATA_DIR, element.type, str(element.id)) + if element.type == "kvm_interface": + element.type = "kvmqm_interface" + if os.path.exists(oldPath): + newPath = os.path.join(config.DATA_DIR, element.type, str(elementMapping[element.id].id)) + cmd_ = ["mv", oldPath, newPath] + out = cmd.run(cmd_) + except Exception, e: + oldPath = os.path.join(config.DATA_DIR, element.type, str(element.id)) + if element.type == "kvm_interface": + element.type = "kvmqm_interface" + newPath = os.path.join(config.DATA_DIR, element.type, str(elementMapping[element.id].id)) + print "Couldn't move element folder: old_id=%s, new_id=%s, type=%s, path=%s, new_path=%s" \ + % (str(element.id), + str(elementMapping[element.id].id), + str(element.type), + str(oldPath), + str(newPath)) + print e + + for connection in Connection.objects.filter(): + try: + oldPath = os.path.join(config.DATA_DIR, connection.type, str(connection.id)) + if os.path.exists(oldPath): + newPath = os.path.join(config.DATA_DIR, connection.type, str(connectionMapping[connection.id].id)) + cmd_ = ["mv", oldPath, newPath] + out = cmd.run(cmd_) + except Exception, e: + oldPath = os.path.join(config.DATA_DIR, connection.type, str(connection.id)) + newPath = os.path.join(config.DATA_DIR, connection.type, str(connectionMapping[connection.id].id)) + print "Couldn't move connection folder: old_id=%s, new_id=%s, type=%s, path=%s, new_path=%s" \ + % (str(connection.id), + str(connectionMapping[connection.id].id), + str(connection.type), + str(oldPath), + str(newPath)) + raise e + """ diff --git a/hostmanager/tomato/elements/__init__.py b/hostmanager/tomato/elements/__init__.py index 13ed864d5..25d2cb751 100644 --- a/hostmanager/tomato/elements/__init__.py +++ b/hostmanager/tomato/elements/__init__.py @@ -141,7 +141,7 @@ def dataPath(self, filename=""): @param filename: a filename relative to the data path @type filename: str """ - return os.path.join(config.DATA_DIR, self.TYPE, str(self.id), filename) + return os.path.join(config.DATA_DIR, self.TYPE, str(self.getId()), filename) def upcast(self): """ @@ -163,6 +163,20 @@ class that it possesses. Due to a limitation of the database backend, def hasParent(self): return not self.parent is None + def getId(self): + """ + Returns the element id, converts automatically back to old postgres IDs if possible + :return: String + """ + id_ = str(self.id) + try: + #As we converted old ids by adding zeros to the left, we can check if the first 12 characters are zeros + if id_[0:12] == "000000000000": + id_ = int(id_, 16) + except: + pass + return id_ + def checkModify(self, attrs): """ Checks whether the attribute change can succeed before changing the @@ -331,7 +345,7 @@ def cap_attrs(cls): def info(self): res = { - "id": str(self.id), + "id": str(self.getId()), "type": self.type, "parent": str(self.parent.id) if self.hasParent() else None, "state": self.state, diff --git a/hostmanager/tomato/elements/kvm.py b/hostmanager/tomato/elements/kvm.py index 8f18b59cd..596f1abf3 100644 --- a/hostmanager/tomato/elements/kvm.py +++ b/hostmanager/tomato/elements/kvm.py @@ -138,7 +138,7 @@ class KVM(elements.Element, elements.RexTFVElement): vncport = IntField() vncpid = IntField() vncpassword =StringField() - cpus = IntField(default=1) + cpus = FloatField(default=1) ram = IntField(default=256) kblang = StringField(default=None, choises=kblang_options) usbtablet = BooleanField(default=True) @@ -359,27 +359,31 @@ def _imagePath(self, file="disk.qcow2"): #@ReservedAssignment return os.path.join(self._imagePathDir(), file) def updateUsage(self, usage, data): - self._checkState() - if self.state == StateName.CREATED: - return - if self.state == StateName.STARTED: - memory = 0 - cputime = 0 - stats = self.vir.getStatistics(self.vmid) - memory += stats.memory_used - cputime += stats.cputime_total - if self.vncpid and proc.isAlive(self.vncpid): - stats = proc.getStatistics(self.vncpid) - memory += stats.memory_used - cputime += stats.cputime_total - if self.websocket_pid and proc.isAlive(self.websocket_pid): - stats = proc.getStatistics(self.websocket_pid) + try: + self._checkState() + if self.state == StateName.CREATED: + return + if self.state == StateName.STARTED: + memory = 0 + cputime = 0 + stats = self.vir.getStatistics(self.vmid) memory += stats.memory_used cputime += stats.cputime_total - usage.memory = memory - usage.updateContinuous("cputime", cputime, data) - usage.diskspace = io.getSize(self._imagePathDir()) - + if self.vncpid and proc.isAlive(self.vncpid): + stats = proc.getStatistics(self.vncpid) + memory += stats.memory_used + cputime += stats.cputime_total + if self.websocket_pid and proc.isAlive(self.websocket_pid): + stats = proc.getStatistics(self.websocket_pid) + memory += stats.memory_used + cputime += stats.cputime_total + usage.memory = memory + usage.updateContinuous("cputime", cputime, data) + usage.diskspace = io.getSize(self._imagePathDir()) + except InternalError.WRONG_DATA: + pass + except: + raise """ RexTFV """ @@ -527,7 +531,6 @@ def _nlxtp_create_device_and_mountpoint(self): # if device file or mount point class KVM_Interface(elements.Element):# num = IntField() - name = StringField() mac = StringField() ipspy_pid = IntField() used_addresses = ListField(default=[]) @@ -604,7 +607,6 @@ def updateUsage(self, usage, data): "num": Attribute(field=num, schema=schema.Int(), readOnly=True), "mac": Attribute(field=mac, description="Mac Address", schema=schema.String(), readOnly=True), "ipspy_id": Attribute(field=ipspy_pid, schema=schema.Int(), readOnly=True), - "name": Attribute(field=name, description="Name", schema=schema.String(regex="^eth[0-9]+$")), "used_addresses": Attribute(field=used_addresses, schema=schema.List(), default=[], readOnly=True), }) diff --git a/hostmanager/tomato/elements/kvmqm.py b/hostmanager/tomato/elements/kvmqm.py index 9de059785..6be4456a8 100644 --- a/hostmanager/tomato/elements/kvmqm.py +++ b/hostmanager/tomato/elements/kvmqm.py @@ -438,26 +438,31 @@ def info(self): return info def updateUsage(self, usage, data): - self._checkState() - if self.state == StateName.CREATED: - return - if self.state == StateName.STARTED: - memory = 0 - cputime = 0 - stats = qm.getStatistics(self.vmid) - memory += stats.memory_used - cputime += stats.cputime_total - if self.vncpid and proc.isAlive(self.vncpid): - stats = proc.getStatistics(self.vncpid) - memory += stats.memory_used - cputime += stats.cputime_total - if self.websocket_pid and proc.isAlive(self.websocket_pid): - stats = proc.getStatistics(self.websocket_pid) + try: + self._checkState() + if self.state == StateName.CREATED: + return + if self.state == StateName.STARTED: + memory = 0 + cputime = 0 + stats = qm.getStatistics(self.vmid) memory += stats.memory_used cputime += stats.cputime_total - usage.memory = memory - usage.updateContinuous("cputime", cputime, data) - usage.diskspace = io.getSize(self._imagePathDir()) + if self.vncpid and proc.isAlive(self.vncpid): + stats = proc.getStatistics(self.vncpid) + memory += stats.memory_used + cputime += stats.cputime_total + if self.websocket_pid and proc.isAlive(self.websocket_pid): + stats = proc.getStatistics(self.websocket_pid) + memory += stats.memory_used + cputime += stats.cputime_total + usage.memory = memory + usage.updateContinuous("cputime", cputime, data) + usage.diskspace = io.getSize(self._imagePathDir()) + except InternalError.WRONG_DATA: + pass + except: + raise ATTRIBUTES = elements.Element.ATTRIBUTES.copy() ATTRIBUTES.update({ @@ -526,8 +531,8 @@ class KVMQM_Interface(elements.Element): num = IntField() - name = StringField() mac = StringField() + name = StringField() ipspy_pid = IntField() used_addresses = ListField(default=[]) @@ -599,8 +604,8 @@ def updateUsage(self, usage, data): ATTRIBUTES.update({ "num": Attribute(field=num, schema=schema.Int(), readOnly=True), "mac": Attribute(field=mac, description="Mac Address", schema=schema.String(), readOnly=True), - "ipspy_id": Attribute(field=ipspy_pid, schema=schema.Int(), readOnly=True), "name": Attribute(field=name, description="Name", schema=schema.String(regex="^eth[0-9]+$")), + "ipspy_id": Attribute(field=ipspy_pid, schema=schema.Int(), readOnly=True), "used_addresses": Attribute(field=used_addresses, schema=schema.List(), default=[], readOnly=True), }) diff --git a/hostmanager/tomato/elements/openvz.py b/hostmanager/tomato/elements/openvz.py index 717ffd7a9..12754eab7 100644 --- a/hostmanager/tomato/elements/openvz.py +++ b/hostmanager/tomato/elements/openvz.py @@ -155,7 +155,7 @@ class OpenVZ(elements.Element, elements.RexTFVElement): vncport = IntField() vncpid = IntField() vncpassword =StringField() - cpus = IntField(default=1) + cpus = FloatField(default=1) ram = IntField(default=256) diskspace = IntField(default=10240) rootpassword = StringField() @@ -189,7 +189,7 @@ def init(self, *args, **kwargs): self.vncport = self.getResource("port") self.websocket_port = self.getResource("port", config.WEBSOCKIFY_PORT_BLACKLIST) self.vncpassword = cmd.randomPassword() - self.update_or_save() + self.update_or_save(vmid=self.vmid, vncport=self.vncport, websocket_port=self.websocket_port, vncpassword=self.vncpassword) #template: None, default template def _imagePath(self): @@ -596,18 +596,23 @@ def _diskspace(self): path.diskspace(self._imagePath()) def updateUsage(self, usage, data): - self._checkState() - if self.state == StateName.CREATED: - return - cputime = self._cputime() - if cputime: - usage.updateContinuous("cputime", cputime, data) - memory = self._memory() - if memory: - usage.memory = memory - diskspace = self._diskspace() - if diskspace: - usage.diskspace = diskspace + try: + self._checkState() + if self.state == StateName.CREATED: + return + cputime = self._cputime() + if cputime: + usage.updateContinuous("cputime", cputime, data) + memory = self._memory() + if memory: + usage.memory = memory + diskspace = self._diskspace() + if diskspace: + usage.diskspace = diskspace + except InternalError.WRONG_DATA: + pass + except: + raise ATTRIBUTES = elements.Element.ATTRIBUTES.copy() @@ -619,7 +624,7 @@ def updateUsage(self, usage, data): "vncpid": Attribute(field=vncpid, readOnly=True, schema=schema.Int()), "vncpassword": Attribute(field=vncpassword, readOnly=True, schema=schema.String(null=True)), "hostname": Attribute(field=hostname, label="Hostname", set=modify_hostname, schema=schema.String(null=True)), - "cpus": Attribute(field=cpus, label="Number of CPUs", schema=schema.Number(minValue=1,maxValue=4), default=1), + "cpus": Attribute(field=cpus, label="Number of CPUs", schema=schema.Number(minValue=0.1, maxValue=4.0), default=1.0), "ram": Attribute(field=ram, label="RAM", schema=schema.Int(minValue=64, maxValue=8192), default=256), "diskspace": Attribute(field=diskspace, label="Disk space in MB", schema=schema.Int(minValue=512, maxValue=102400), default=10240), "rootpassword": Attribute(field=rootpassword, label="Root password", schema=schema.String(null=True)), diff --git a/hostmanager/tomato/elements/repy.py b/hostmanager/tomato/elements/repy.py index 03f8263e1..51f02d5e9 100644 --- a/hostmanager/tomato/elements/repy.py +++ b/hostmanager/tomato/elements/repy.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see -import os, sys, shutil +import os, sys, time, shutil from ..db import * from ..generic import * from .. import connections, elements, config @@ -127,7 +127,7 @@ class Repy(elements.Element): vncpassword =StringField() args = ListField() args_doc = StringField(default=None, required=False) - cpus = IntField(default=1) + cpus = FloatField(default=1) ram = IntField(default=256) bandwidth = IntField(min_value=1024, max_value=10000000000, default=1000000) template = ReferenceField(template.Template, null=True) @@ -150,8 +150,14 @@ def type(self): return self.TYPE def init(self, *args, **kwargs): + self.state = StateName.PREPARED - elements.Element.init(self, *args, **kwargs) #no id and no attrs before this line + self.timeout = time.time() + config.MAX_TIMEOUT + #Workaround. We need repy to have a folder and be saved befor we start setting attributes. + self.update_or_save() + if not os.path.exists(self.dataPath()): + os.makedirs(self.dataPath()) + elements.Element.init(self, *args, **kwargs) self.vncport = self.getResource("port") self.websocket_port = self.getResource("port", config.WEBSOCKIFY_PORT_BLACKLIST) self.vncpassword = cmd.randomPassword() @@ -324,15 +330,20 @@ def info(self): return info def updateUsage(self, usage, data): - self._checkState() - usage.diskspace = path.diskspace(self.dataPath()) - if self.state == StateName.STARTED: - usage.memory = process.memory(self.pid) - cputime = process.cputime(self.pid) - if self.vncpid: - usage.memory += process.memory(self.vncpid) - cputime += process.cputime(self.vncpid) - usage.updateContinuous("cputime", cputime, data) + try: + self._checkState() + usage.diskspace = path.diskspace(self.dataPath()) + if self.state == StateName.STARTED: + usage.memory = process.memory(self.pid) + cputime = process.cputime(self.pid) + if self.vncpid: + usage.memory += process.memory(self.vncpid) + cputime += process.cputime(self.vncpid) + usage.updateContinuous("cputime", cputime, data) + except InternalError.WRONG_DATA: + pass + except: + raise ATTRIBUTES = elements.Element.ATTRIBUTES.copy() ATTRIBUTES.update({ @@ -344,8 +355,8 @@ def updateUsage(self, usage, data): "vncpassword": Attribute(field=vncpassword, readOnly=True, schema=schema.String()), "args": Attribute(field=args, description="Arguments", set=modify_args, schema = schema.List(), default=[]), "args_doc": Attribute(field=args_doc, description="Arguments Documentation", readOnly=True), - "cpus": Attribute(field=cpus, description="Number of CPUs", set=modify_cpus, schema=schema.Number(minValue=1,maxValue=4), default=1), - "ram": Attribute(field=ram, description="RAM", set=modify_ram, schema=schema.Int(minValue=64, maxValue=8192), default=256), + "cpus": Attribute(field=cpus, description="Number of CPUs", set=modify_cpus, schema=schema.Number(minValue=0.011,maxValue=4.0), default=0.25), + "ram": Attribute(field=ram, description="RAM", set=modify_ram, schema=schema.Int(minValue=10, maxValue=8192), default=25), "bandwidth": Attribute(field=bandwidth, description="Bandwidth in bytes/s", set=modify_bandwidth, schema=schema.Int(minValue=1024, maxValue=10000000000), default=1000000), "template": Attribute(field=templateId, description="Template", set=modify_template, schema=schema.Identifier()), }) @@ -431,10 +442,13 @@ def type(self): return self.TYPE def init(self, *args, **kwargs): + print "Init a repy interface element" self.state = StateName.PREPARED + print "State is set, now activate Element.Init for repy interface" elements.Element.init(self, *args, **kwargs) #no id and no attrs before this line assert isinstance(self.getParent(), Repy) self.name = self.getParent()._nextIfaceName() + self.update_or_save(name=self.name) def interfaceName(self): return self.getParent()._interfaceName(self.name) diff --git a/hostmanager/tomato/resources/__init__.py b/hostmanager/tomato/resources/__init__.py index ae8e7cc2b..9b3cdfc3e 100644 --- a/hostmanager/tomato/resources/__init__.py +++ b/hostmanager/tomato/resources/__init__.py @@ -96,7 +96,22 @@ def modify(self, attrs): self.attrs[key] = value logging.logMessage("info", category="resource", type=self.type, id=str(self.id), info=self.info()) self.update_or_save() - + + def getId(self): + """ + Returns the element id, converts automatically back to old postgres IDs if possible + :return: String + """ + id_ = self.id + try: + #As we converted old ids by adding zeros to the left, we can check if the first 12 characters are zeros + if self.id[0:12] == "000000000000": + oldId = int(id_, 16) + id_ = "{0:0{1}x}".format(oldId, 24) + except: + pass + return id_ + def remove(self): logging.logMessage("info", category="resource", type=self.type, id=str(self.id), info=self.info()) logging.logMessage("remove", category="resource", type=self.type, id=str(self.id)) @@ -104,7 +119,7 @@ def remove(self): def info(self): return { - "id": str(self.id), + "id": str(self.getId()), "type": self.type, "attrs": self.attrs.copy(), } @@ -118,15 +133,6 @@ class ResourceInstance(BaseDocument): ownerConnectionId = ReferenceFieldId(ownerConnection) attrs = DictField() - ACTIONS = {} - ATTRIBUTES = { - "id": IdAttribute(), - "type": Attribute(field=type, schema=schema.String(options=TYPES)), - "num": Attribute(field=num, schema=schema.Int(minValue=0)), - "ownerElement": Attribute(field=ownerElementId, schema=schema.Identifier()), - "ownerConnection": Attribute(field=ownerConnectionId, schema=schema.Identifier()), - } - def init(self, type, num, owner, attrs=None): #@ReservedAssignment if not attrs: attrs = {} self.type = type @@ -141,6 +147,29 @@ def init(self, type, num, owner, attrs=None): #@ReservedAssignment self.attrs = attrs self.save() + def getId(self): + """ + Returns the element id, converts automatically back to old postgres IDs if possible + :return: String + """ + id_ = str(self.id) + try: + #As we converted old ids by adding zeros to the left, we can check if the first 12 characters are zeros + if id_[0:12] == "000000000000": + id_ = int(id_, 16) + except: + pass + return id_ + + ACTIONS = {} + ATTRIBUTES = { + "id": IdAttribute(), + "type": Attribute(field=type, schema=schema.String(options=TYPES)), + "num": Attribute(field=num, schema=schema.Int(minValue=0)), + "ownerElement": Attribute(field=ownerElementId, schema=schema.Identifier()), + "ownerConnection": Attribute(field=ownerConnectionId, schema=schema.Identifier()), + } + def get(id_, **kwargs): try: