Statistics
| Branch: | Tag: | Revision:

one / src / onedb / local / 4.13.85_to_4.90.0.rb @ 221ce1c6

History | View | Annotate | Download (33.3 KB)

1
# -------------------------------------------------------------------------- #
2
# Copyright 2002-2017, OpenNebula Project, OpenNebula Systems                #
3
#                                                                            #
4
# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
5
# not use this file except in compliance with the License. You may obtain    #
6
# a copy of the License at                                                   #
7
#                                                                            #
8
# http://www.apache.org/licenses/LICENSE-2.0                                 #
9
#                                                                            #
10
# Unless required by applicable law or agreed to in writing, software        #
11
# distributed under the License is distributed on an "AS IS" BASIS,          #
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
13
# See the License for the specific language governing permissions and        #
14
# limitations under the License.                                             #
15
#--------------------------------------------------------------------------- #
16

    
17
require 'set'
18
require 'base64'
19
require 'zlib'
20
require 'pathname'
21

    
22
require 'opennebula'
23

    
24
include OpenNebula
25

    
26
module Migrator
27
  def db_version
28
    "4.90.0"
29
  end
30

    
31
  def one_version
32
    "OpenNebula 4.90.0"
33
  end
34

    
35
  TEMPLATE_TRANSFORM_ATTRS = {
36
    'SUNSTONE_NETWORK_SELECT'   => 'NETWORK_SELECT'
37
  }
38

    
39
  def up
40
    init_log_time()
41

    
42
    ############################################################################
43
    # 4369
44
    ############################################################################
45

    
46
    @db.run "CREATE TABLE cluster_datastore_relation (cid INTEGER, oid INTEGER, PRIMARY KEY(cid, oid));"
47
    @db.run "CREATE TABLE cluster_network_relation (cid INTEGER, oid INTEGER, PRIMARY KEY(cid, oid));"
48

    
49

    
50
    @db.run "ALTER TABLE host_pool RENAME TO old_host_pool;"
51
    @db.run "CREATE TABLE host_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, state INTEGER, last_mon_time INTEGER, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, cid INTEGER);"
52

    
53
    host_ids = []
54

    
55
    @db.transaction do
56
      @db.fetch("SELECT * FROM old_host_pool") do |row|
57
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
58

    
59
        cid_elem = doc.root.at_xpath("CLUSTER_ID")
60
        cid = cid_elem.text.to_i
61

    
62
        if (cid == -1)
63
          cid = 0
64
          cid_elem.content = "0"
65
          doc.root.at_xpath("CLUSTER").content = "default"
66

    
67
          host_ids << row[:oid]
68
        end
69

    
70
        @db[:host_pool].insert(
71
          :oid            => row[:oid],
72
          :name           => row[:name],
73
          :body           => doc.root.to_s,
74
          :state          => row[:state],
75
          :last_mon_time  => row[:last_mon_time],
76
          :uid            => row[:uid],
77
          :gid            => row[:gid],
78
          :owner_u        => row[:owner_u],
79
          :group_u        => row[:group_u],
80
          :other_u        => row[:other_u],
81
          :cid            => cid)
82
      end
83
    end
84

    
85
    @db.run "DROP TABLE old_host_pool;"
86

    
87
    log_time()
88

    
89
    @db.run "ALTER TABLE datastore_pool RENAME TO old_datastore_pool;"
90
    @db.run "CREATE TABLE datastore_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);"
91

    
92
    ds_ids = []
93

    
94
    @db.transaction do
95
      @db.fetch("SELECT * FROM old_datastore_pool") do |row|
96
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
97

    
98
        doc.root.at_xpath("CLUSTER").remove
99

    
100
        cid_elem = doc.root.at_xpath("CLUSTER_ID")
101
        cid = cid_elem.text.to_i
102

    
103
        cid_elem.remove
104

    
105
        if (cid == -1)
106
          cid = 0
107
          ds_ids << row[:oid]
108
        end
109

    
110
        cluster_ids_elem = doc.create_element("CLUSTERS")
111
        cluster_ids_elem.add_child(doc.create_element("ID")).content = cid.to_s
112

    
113
        doc.root.add_child(cluster_ids_elem)
114

    
115
        @db[:datastore_pool].insert(
116
          :oid      => row[:oid],
117
          :name     => row[:name],
118
          :body     => doc.root.to_s,
119
          :uid      => row[:uid],
120
          :gid      => row[:gid],
121
          :owner_u  => row[:owner_u],
122
          :group_u  => row[:group_u],
123
          :other_u  => row[:other_u])
124

    
125
        @db[:cluster_datastore_relation].insert(
126
          :cid => cid,
127
          :oid => row[:oid])
128
      end
129
    end
130

    
131
    @db.run "DROP TABLE old_datastore_pool;"
132

    
133
    log_time()
134

    
135
    @db.run "ALTER TABLE network_pool RENAME TO old_network_pool;"
136
    @db.run "CREATE TABLE network_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, pid INTEGER, UNIQUE(name,uid));"
137

    
138
    vnet_ids = []
139

    
140
    @db.transaction do
141
      @db.fetch("SELECT * FROM old_network_pool") do |row|
142
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
143

    
144
        doc.root.at_xpath("CLUSTER").remove
145

    
146
        cid_elem = doc.root.at_xpath("CLUSTER_ID")
147
        cid = cid_elem.text.to_i
148

    
149
        cid_elem.remove
150

    
151
        if (cid == -1)
152
          cid = 0
153
          vnet_ids << row[:oid]
154
        end
155

    
156
        cluster_ids_elem = doc.create_element("CLUSTERS")
157
        cluster_ids_elem.add_child(doc.create_element("ID")).content = cid.to_s
158

    
159
        doc.root.add_child(cluster_ids_elem)
160

    
161
        @db[:network_pool].insert(
162
          :oid      => row[:oid],
163
          :name     => row[:name],
164
          :body     => doc.root.to_s,
165
          :uid      => row[:uid],
166
          :gid      => row[:gid],
167
          :owner_u  => row[:owner_u],
168
          :group_u  => row[:group_u],
169
          :other_u  => row[:other_u],
170
          :pid      => row[:pid])
171

    
172
        @db[:cluster_network_relation].insert(
173
          :cid => cid,
174
          :oid => row[:oid])
175
      end
176
    end
177

    
178
    @db.run "DROP TABLE old_network_pool;"
179

    
180
    log_time()
181

    
182
    @db.run "ALTER TABLE vm_pool RENAME TO old_vm_pool;"
183
    @db.run "CREATE TABLE vm_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, last_poll INTEGER, state INTEGER, lcm_state INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER)"
184

    
185
    @db.transaction do
186
      @db.fetch("SELECT * FROM old_vm_pool") do |row|
187
        if row[:state] != 6
188
          doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
189

    
190
          cid = doc.root.at_xpath("HISTORY_RECORDS/HISTORY[last()]/CID").text.to_i rescue nil
191

    
192
          if cid == -1
193
            doc.root.at_xpath("HISTORY_RECORDS/HISTORY[last()]/CID").content = 0
194
          end
195

    
196
          # Bug #4467
197

    
198
          elem = doc.at_xpath("/VM/USER_TEMPLATE/EC2")
199

    
200
          if (!elem.nil?)
201
            elem.name = "PUBLIC_CLOUD"
202

    
203
            if elem.at_xpath("TYPE").nil?
204
              elem.add_child(doc.create_element("TYPE")).content = "ec2"
205
            end
206
          end
207

    
208
          row[:body] = doc.root.to_s
209
        end
210

    
211
        @db[:vm_pool].insert(row)
212
      end
213
    end
214

    
215
    @db.run "DROP TABLE old_vm_pool;"
216

    
217
    log_time()
218

    
219
    default_cl_xml = '<CLUSTER><ID>0</ID><NAME>default</NAME><HOSTS></HOSTS><DATASTORES></DATASTORES><VNETS></VNETS><TEMPLATE><RESERVED_CPU><![CDATA[]]></RESERVED_CPU><RESERVED_MEM><![CDATA[]]></RESERVED_MEM></TEMPLATE></CLUSTER>'
220
    doc = Nokogiri::XML(default_cl_xml,nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
221

    
222
    hosts_elem = doc.root.at_xpath("HOSTS")
223
    host_ids.each { |id|
224
      hosts_elem.add_child(doc.create_element("ID")).content = id.to_s
225
    }
226

    
227
    ds_elem = doc.root.at_xpath("DATASTORES")
228
    ds_ids.each { |id|
229
      ds_elem.add_child(doc.create_element("ID")).content = id.to_s
230
    }
231

    
232
    vnets_elem = doc.root.at_xpath("VNETS")
233
    vnet_ids.each { |id|
234
      vnets_elem.add_child(doc.create_element("ID")).content = id.to_s
235
    }
236

    
237
    @db[:cluster_pool].insert(
238
      :oid      => 0,
239
      :name     => 'default',
240
      :body     => doc.root.to_s,
241
      :uid      => 0,
242
      :gid      => 0,
243
      :owner_u  => 1,
244
      :group_u  => 0,
245
      :other_u  => 0)
246

    
247
    log_time()
248

    
249
    ############################################################################
250
    # 4215
251
    ############################################################################
252

    
253
    @db.run "CREATE TABLE vrouter_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);"
254

    
255
    @db.run "ALTER TABLE network_pool RENAME TO old_network_pool;"
256
    @db.run "CREATE TABLE network_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, pid INTEGER, UNIQUE(name,uid));"
257

    
258
    @db.transaction do
259
      @db.fetch("SELECT * FROM old_network_pool") do |row|
260
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
261

    
262
        doc.root.add_child(doc.create_element("VROUTERS"))
263

    
264
        @db[:network_pool].insert(
265
          :oid      => row[:oid],
266
          :name     => row[:name],
267
          :body     => doc.root.to_s,
268
          :uid      => row[:uid],
269
          :gid      => row[:gid],
270
          :owner_u  => row[:owner_u],
271
          :group_u  => row[:group_u],
272
          :other_u  => row[:other_u],
273
          :pid      => row[:pid])
274
      end
275
    end
276

    
277
    @db.run "DROP TABLE old_network_pool;"
278

    
279
    log_time()
280

    
281
    @db.run "ALTER TABLE template_pool RENAME TO old_template_pool;"
282
    @db.run "CREATE TABLE template_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);"
283

    
284
    @db.transaction do
285
      @db.fetch("SELECT * FROM old_template_pool") do |row|
286
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
287

    
288
        # Feature #3671
289

    
290
        TEMPLATE_TRANSFORM_ATTRS.each do |old_name, new_name|
291
          elem = doc.at_xpath("/VMTEMPLATE/TEMPLATE/#{old_name}")
292

    
293
          if (!elem.nil?)
294
            elem.remove
295

    
296
            elem.name = new_name
297

    
298
            if (doc.at_xpath("/VMTEMPLATE/TEMPLATE/SUNSTONE").nil?)
299
              doc.at_xpath("/VMTEMPLATE/TEMPLATE").add_child(
300
                doc.create_element("SUNSTONE"))
301
            end
302

    
303
            doc.at_xpath("/VMTEMPLATE/TEMPLATE/SUNSTONE").add_child(elem)
304
          end
305
        end
306

    
307
        # Feature #4317
308

    
309
        elem = doc.at_xpath("/VMTEMPLATE/TEMPLATE/SUNSTONE_CAPACITY_SELECT")
310

    
311
        if elem.nil?
312
          capacity_edit = true
313
        else
314
          elem.remove
315
          capacity_edit = (elem.text != "NO")
316
        end
317

    
318
        if !capacity_edit
319
          cpu_e = doc.at_xpath("/VMTEMPLATE/TEMPLATE/CPU")
320
          memory_e = doc.at_xpath("/VMTEMPLATE/TEMPLATE/MEMORY")
321
          vcpu_e = doc.at_xpath("/VMTEMPLATE/TEMPLATE/VCPU")
322

    
323
          cpu    = cpu_e != nil ? cpu_e.text : ""
324
          memory = memory_e != nil ? memory_e.text : ""
325
          vcpu   = vcpu_e != nil ? vcpu_e.text : ""
326

    
327
          user_inputs = doc.at_xpath("/VMTEMPLATE/TEMPLATE/USER_INPUTS")
328

    
329
          if user_inputs.nil?
330
            user_inputs = doc.create_element("USER_INPUTS")
331
            doc.at_xpath("/VMTEMPLATE/TEMPLATE").add_child(user_inputs)
332
          end
333

    
334
          user_inputs.add_child(doc.create_element("CPU")).content = "O|fixed|||#{cpu}"
335
          user_inputs.add_child(doc.create_element("MEMORY")).content = "O|fixed|||#{memory}"
336
          user_inputs.add_child(doc.create_element("VCPU")).content = "O|fixed|||#{vcpu}"
337
        end
338

    
339
        # Bug #4467
340

    
341
        elem = doc.at_xpath("/VMTEMPLATE/TEMPLATE/EC2")
342

    
343
        if (!elem.nil?)
344
          elem.name = "PUBLIC_CLOUD"
345

    
346
          if elem.at_xpath("TYPE").nil?
347
            elem.add_child(doc.create_element("TYPE")).content = "ec2"
348
          end
349
        end
350

    
351
        @db[:template_pool].insert(
352
          :oid        => row[:oid],
353
          :name       => row[:name],
354
          :body       => doc.root.to_s,
355
          :uid        => row[:uid],
356
          :gid        => row[:gid],
357
          :owner_u    => row[:owner_u],
358
          :group_u    => row[:group_u],
359
          :other_u    => row[:other_u])
360
      end
361
    end
362

    
363
    @db.run "DROP TABLE old_template_pool;"
364

    
365
    log_time()
366

    
367
    ############################################################################
368
    # Feature #4217
369
    ############################################################################
370

    
371
    @db.run "ALTER TABLE image_pool RENAME TO old_image_pool;"
372
    @db.run "CREATE TABLE image_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, UNIQUE(name,uid) );"
373

    
374
    @db.transaction do
375
      @db.fetch("SELECT * FROM old_image_pool") do |row|
376
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
377

    
378
        doc.root.add_child(doc.create_element("APP_CLONES"))
379

    
380
        @db[:image_pool].insert(
381
          :oid        => row[:oid],
382
          :name       => row[:name],
383
          :body       => doc.root.to_s,
384
          :uid        => row[:uid],
385
          :gid        => row[:gid],
386
          :owner_u    => row[:owner_u],
387
          :group_u    => row[:group_u],
388
          :other_u    => row[:other_u])
389
      end
390
    end
391

    
392
    @db.run "DROP TABLE old_image_pool;"
393

    
394
    log_time()
395

    
396
    ############################################################################
397
    # Feature #3204
398
    ############################################################################
399

    
400
    @db.run "ALTER TABLE secgroup_pool RENAME TO old_secgroup_pool;"
401
    @db.run "CREATE TABLE secgroup_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, UNIQUE(name,uid));"
402

    
403
    @db.transaction do
404
      @db.fetch("SELECT * FROM old_secgroup_pool") do |row|
405
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
406

    
407
        doc.root.at_xpath("VMS").name = "UPDATED_VMS"
408
        doc.root.add_child(doc.create_element("OUTDATED_VMS"))
409
        doc.root.add_child(doc.create_element("UPDATING_VMS"))
410
        doc.root.add_child(doc.create_element("ERROR_VMS"))
411

    
412
        @db[:secgroup_pool].insert(
413
          :oid        => row[:oid],
414
          :name       => row[:name],
415
          :body       => doc.root.to_s,
416
          :uid        => row[:uid],
417
          :gid        => row[:gid],
418
          :owner_u    => row[:owner_u],
419
          :group_u    => row[:group_u],
420
          :other_u    => row[:other_u])
421
      end
422
    end
423

    
424
    @db.run "DROP TABLE old_secgroup_pool;"
425

    
426
    log_time()
427

    
428
    # Bug #4248 - Remove Firewall Drivers
429

    
430
    vms_with_fw = []
431
    @db.transaction do
432
      @db.fetch("SELECT * FROM vm_pool WHERE state != 6") do |row|
433
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
434

    
435
        has_fw_attrs = !doc.root.xpath("TEMPLATE/NIC[ICMP|WHITE_PORTS_TCP|WHITE_PORTS_UDP|BLACK_PORTS_TCP|BLACK_PORTS_UDP]").empty?
436

    
437
        vms_with_fw << row[:oid].to_i if has_fw_attrs
438
      end
439
    end
440

    
441
    if !vms_with_fw.empty?
442
      puts "**************************************************************"
443
      puts "*  WARNING  WARNING WARNING WARNING WARNING WARNING WARNING  *"
444
      puts "**************************************************************"
445
      puts
446
      puts "The old driver 'fw' has been removed from OpenNebula. It was  "
447
      puts "deprecated in 4.12:                                           "
448
      puts "http://docs.opennebula.org/4.12/release_notes/release_notes/compatibility.html"
449
      puts
450
      puts "We have detected that you still have active VMs with these    "
451
      puts "attributes: ICMP, WHITE_PORTS_TCP, WHITE_PORTS_UDP,           "
452
      puts "BLACK_PORTS_TCP, BLACK_PORTS_UDP.                             "
453
      puts
454
      puts "The list of affected VMs is:                                  "
455
      vms_with_fw.each{|vm| puts "- #{vm}"}
456
      puts
457
      puts "Please note that OpenNebula will not modify the current       "
458
      puts "iptables rules, so you will need to manually clean them when  "
459
      puts "any of VMs is removed."
460
      puts
461
      puts "Please consider switching to Security Groups                  "
462
    end
463

    
464
    log_time()
465

    
466
    ############################################################################
467
    # Remove Xen, VMware and SoftLayer Drivers
468
    ############################################################################
469

    
470
    @db.run "ALTER TABLE host_pool RENAME TO old_host_pool;"
471

    
472
    @db.run "CREATE TABLE host_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, state INTEGER, last_mon_time INTEGER, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, cid INTEGER);"
473

    
474
    has_xen_hosts    = false
475
    has_vmware_hosts = false
476
    has_sl_hosts     = false
477

    
478
    @db.transaction do
479
      @db.fetch("SELECT * FROM old_host_pool") do |row|
480
        do_disable = false
481

    
482
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
483

    
484
        vm_mad = doc.root.at_xpath("VM_MAD").text
485
        im_mad = doc.root.at_xpath("IM_MAD").text
486

    
487
        if vm_mad.match(/xen/) || im_mad.match(/xen/)
488
          do_disable    = true
489
          has_xen_hosts = true
490
        end
491

    
492
        if vm_mad.match(/vmware/) || im_mad.match(/vmware/)
493
          do_disable       = true
494
          has_vmware_hosts = true
495
        end
496

    
497

    
498
        if vm_mad.match(/sl/) || im_mad.match(/sl/)
499
          do_disable       = true
500
          has_sl_hosts = true
501
        end
502

    
503
        if do_disable
504
          doc.root.at_xpath('STATE').content = 4
505

    
506
          row[:state] = 4
507
          row[:body]  = doc.root.to_s
508
        end
509

    
510
        @db[:host_pool].insert(row)
511
      end
512
    end
513

    
514
    @db.run "DROP TABLE old_host_pool;"
515

    
516
    if has_xen_hosts
517
      puts "**************************************************************"
518
      puts "*  WARNING  WARNING WARNING WARNING WARNING WARNING WARNING  *"
519
      puts "**************************************************************"
520
      puts
521
      puts "Xen is no longer included in the core distribution. It is"
522
      puts "however available as an addon which must be manually installed:"
523
      puts "https://github.com/OpenNebula/addon-xen"
524
      puts
525
      puts "Note that the host has been automatically disabled. After installing"
526
      puts "the addon you can manually enable it."
527
      puts
528
    end
529

    
530
    if has_vmware_hosts
531
      puts "**************************************************************"
532
      puts "*  WARNING  WARNING WARNING WARNING WARNING WARNING WARNING  *"
533
      puts "**************************************************************"
534
      puts
535
      puts "VMware is no longer supported. You are encouraged to migrate"
536
      puts "to the vCenter driver:"
537
      puts "http://docs.opennebula.org/stable/administration/virtualization/vcenterg.html"
538
      puts
539
      puts "Note that the host has been automatically disabled, but not removed."
540
      puts
541
    end
542

    
543
    if has_sl_hosts
544
      puts "**************************************************************"
545
      puts "*  WARNING  WARNING WARNING WARNING WARNING WARNING WARNING  *"
546
      puts "**************************************************************"
547
      puts
548
      puts "SoftLayer is no longer included in the core distribution. It is"
549
      puts "however available as an addon which must be manually installed:"
550
      puts "https://github.com/OpenNebula/addon-softlayer"
551
      puts
552
      puts "Note that the host has been automatically disabled. After installing"
553
      puts "the addon you can manually enable it."
554
      puts
555
    end
556

    
557
    log_time()
558

    
559
    ############################################################################
560
    # Move HOST/VN_MAD --> VNET/VN_MAD
561
    ############################################################################
562

    
563
    # Build net_vnmad
564
    net_vnmad = {}
565
    @db.transaction do
566
      @db.fetch("SELECT * FROM vm_pool WHERE state != 6") do |row|
567
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
568
        state = row[:state].to_i
569

    
570
        vnmads = Set.new
571
        doc.root.xpath("HISTORY_RECORDS/HISTORY/VNMMAD").collect{|v| vnmads << v.text }
572

    
573
        doc.root.xpath("TEMPLATE/NIC/NETWORK_ID").each do |net_id|
574
            net_id = net_id.text.to_i
575

    
576
            net_vnmad[net_id] ||= Set.new
577
            net_vnmad[net_id] += vnmads
578
        end
579
      end
580
    end
581

    
582
    # Build cluster_vnmad and fix hosts
583
    @db.run "ALTER TABLE host_pool RENAME TO old_host_pool;"
584
    @db.run "CREATE TABLE host_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, state INTEGER, last_mon_time INTEGER, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, cid INTEGER);"
585

    
586
    cluster_vnmad = {}
587
    @db.transaction do
588
      @db.fetch("SELECT * FROM old_host_pool") do |row|
589
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
590

    
591
        # Get cluster
592
        cluster_id = doc.root.xpath('CLUSTER_ID').text.to_i
593

    
594
        # Store VN_MAD
595
        vnmad = doc.root.xpath('VN_MAD').text
596

    
597
        cluster_vnmad[cluster_id] ||= Set.new
598
        cluster_vnmad[cluster_id] << vnmad
599

    
600
        # Remove VN_MAD
601
        doc.root.xpath('//VN_MAD').remove
602

    
603
        row[:body] = doc.root.to_s
604
        @db[:host_pool].insert(row)
605
      end
606
    end
607
    @db.run "DROP TABLE old_host_pool;"
608

    
609
    # Fix Networks
610

    
611
    # So far we have two hashes: net_vnmad, which lists the specific vnmad found
612
    # for each network, based on the nics of the VMs, and cluster_vnmad, with
613
    # the vnmad found in the hosts.
614
    #
615
    # We will report a warning if either the cluster or the network has more
616
    # than one vnmad. We will automatically choose one vnmad from the network
617
    # list, or if empty from the cluster list.
618
    #
619
    # That vnmad may be changed through onedb patch.
620
    #
621
    # It could happen that no network is found in the net or in the cluster, in
622
    # which case the admin **must** run the onedb patch, it's not optional any
623
    # more.
624

    
625
    @db.run "ALTER TABLE network_pool RENAME TO old_network_pool;"
626
    @db.run "CREATE TABLE network_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, pid INTEGER, UNIQUE(name,uid));"
627

    
628
    reserved_vlan_ids = Set.new
629
    final_net_vnmad = {}
630
    manual_intervention = false
631
    @db.transaction do
632
      @db.fetch("SELECT * FROM old_network_pool") do |row|
633
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
634

    
635
        net_id   = row[:oid]
636
        net_name = row[:name]
637

    
638
        cluster_id = doc.root.xpath('CLUSTERS/ID').first.text.to_i
639

    
640
        # Get possible VN_MADs
641
        net_vnmad_len     = net_vnmad[net_id].length rescue 0
642
        cluster_vnmad_len = cluster_vnmad[cluster_id].length rescue 0
643

    
644
        # Check if the network has VLAN=NO
645
        vlan = doc.root.xpath('VLAN').text rescue nil
646
        vlan_no = (vlan == "0")
647

    
648
        # Remove the VLAN attributes
649
        doc.root.xpath('//VLAN').remove
650

    
651
        vnmad = nil
652
        other_vnmads = nil
653

    
654
        # Get vnmad
655
        #
656
        # net_vnmad_len == 1 => that one
657
        # net_vnmad_len > 1 => interactive
658
        # net_vnmad_len == 0 && cluster_vnmad_len == 1 => that one
659
        # net_vnmad_len == 0 && cluster_vnmad_len > 1 => interactive
660
        # net_vnmad_len == 0 && cluster_vnmad_len == 0 => interactive
661

    
662
        if net_vnmad_len == 1
663
          vnmad = net_vnmad[net_id].first
664
        elsif net_vnmad_len > 1
665
          other_vnmads = net_vnmad[net_id]
666
        elsif net_vnmad_len == 0 && cluster_vnmad_len == 1
667
          vnmad = cluster_vnmad[cluster_id].first
668
        elsif net_vnmad_len == 0 && cluster_vnmad_len > 1
669
          other_vnmads = cluster_vnmad[cluster_id]
670
        end
671

    
672
        # Ambiguous vnmad, require user input (TODO)
673
        if vnmad.nil?
674

    
675
          if !manual_intervention
676
            manual_intervention = true
677
            puts
678
            puts  "Manual Intervention required. Please input the VN_MAD " <<
679
                  " for the following networks:"
680
            puts
681
          end
682

    
683
          suggested = if other_vnmads
684
            " (suggested: [#{other_vnmads.to_a.join(', ')}])"
685
          else
686
            ""
687
          end
688

    
689
          input = ""
690
          while (input.empty?) do
691
            puts "* Net ##{net_id} (#{net_name}) VN_MAD#{suggested}:"
692
            input = STDIN.gets.chomp.strip
693

    
694
            if input.match(/[^\w\-.]/)
695
              puts "Invalid char found."
696
              input = ""
697
            end
698
          end
699

    
700
          vnmad = input
701
        end
702

    
703
        # If VLAN = NO => don't use isolated VN_MADs
704
        if vlan_no && vnmad && ["802.1q", "ovswitch", "vxlan", "ebtables"].include?(vnmad.downcase)
705
            input = ""
706
            while (input.empty?) do
707
              puts "Net ##{net_id} (#{net_name}) has VN_MAD='#{vnmad}' but it also has VLAN=NO. Change to 'fw'? (y/n)"
708

    
709
              input = STDIN.gets.chomp.strip
710

    
711
              case input
712
              when 'y'
713
                vnmad = 'fw'
714
              when 'n'
715
              else
716
                puts "Invalid value."
717
                input = ""
718
              end
719
            end
720
        end
721

    
722
        if vnmad.nil?
723
          STDERR.puts "Error getting VN_MAD for Network #{net_id}."
724
          exit 1
725
        end
726

    
727
        # Create the VN_MAD element:
728
        final_net_vnmad[net_id] = vnmad
729
        doc.root.add_child(doc.create_element("VN_MAD")).content = vnmad
730

    
731
        # Create/Move the VLAN_ID and VLAN_ID_AUTOMATIC attributes:
732
        #
733
        # Manual    => VLAN_ID exists
734
        # Automatic => VLAN_ID does not exist
735
        #
736
        # If VLAN has been set automatically,
737
        #
738
        #   top-level <VLAN_ID><![CDATA[20]]></VLAN_ID>
739
        #   top-level <VLAN_ID_AUTOMATIC>1</VLAN_ID_AUTOMATIC>
740
        #   remove VLAN_ID from <TEMPLATE>
741
        #
742
        # If VLAN has been set manually,
743
        #
744
        #   top-level <VLAN_ID><![CDATA[20]]></VLAN_ID>
745
        #   top-level <VLAN_ID_AUTOMATIC>0</VLAN_ID_AUTOMATIC>
746
        #   keep VLAN_ID in <TEMPLATE>
747
        #
748
        if vnmad && ["802.1q", "ovswitch", "vxlan"].include?(vnmad.downcase)
749
          vlan_id = doc.root.xpath('TEMPLATE/VLAN_ID').text rescue nil
750

    
751
          if vlan_id && !vlan_id.empty?
752
            vlan_id_automatic = false
753
          else
754
            # TODO: get from configuration?
755
            start_vlan = 2
756

    
757
            vlan_id = start_vlan + (net_id % (4095 - start_vlan))
758
            vlan_id_automatic = true
759

    
760
            # Only automatic vlans will be reserved
761
            if ["802.1q", "ovswitch"].include?(vnmad.downcase)
762
              reserved_vlan_ids << vlan_id
763
            end
764

    
765
            doc.root.xpath("//VLAN_ID[not(parent::AR)]").each {|e| e.remove }
766
          end
767

    
768
          doc.root.add_child(doc.create_element("VLAN_ID")).content = vlan_id
769
          doc.root.add_child(doc.create_element("VLAN_ID_AUTOMATIC")).content = vlan_id_automatic ? "1" : "0"
770
        end
771

    
772
        row[:body] = doc.root.to_s
773
        @db[:network_pool].insert(row)
774
      end
775
    end
776

    
777
    @db.run "DROP TABLE old_network_pool;"
778

    
779
    # Fix VMs
780

    
781
    @db.run "ALTER TABLE vm_pool RENAME TO old_vm_pool;"
782
    @db.run "CREATE TABLE vm_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, last_poll INTEGER, state INTEGER, lcm_state INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER)"
783

    
784
    @db.transaction do
785
      @db.fetch("SELECT * FROM old_vm_pool") do |row|
786
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
787
        state = row[:state].to_i
788

    
789
        if state != 6
790
          # Remove vnmad from the history records
791
          doc.root.xpath("HISTORY_RECORDS//VNMMAD").remove
792

    
793
          # Rename VMMMAD -> VM_MAD and TMMAD -> TM_MAD
794
          doc.root.xpath("HISTORY_RECORDS//VMMMAD").each {|e| e.name = "VM_MAD"}
795
          doc.root.xpath("HISTORY_RECORDS//TMMAD").each  {|e| e.name = "TM_MAD"}
796

    
797
          # Add vnmad to the nics
798
          doc.root.xpath('TEMPLATE/NIC').each do |nic|
799
            net_id = nic.xpath("NETWORK_ID").text.to_i rescue nil # NICs without network may exist
800

    
801
            next unless net_id
802

    
803
            vnmad = final_net_vnmad[net_id]
804

    
805
            if vnmad
806
                nic.add_child(doc.create_element("VN_MAD")).content = vnmad
807
            end
808
          end
809

    
810
          # Remove DS_LOCATION (Feature #4316 - Remove BASE_PATH)
811
          doc.root.xpath("HISTORY_RECORDS//DS_LOCATION").remove
812

    
813
          row[:body] = doc.root.to_s
814
        end
815

    
816
        @db[:vm_pool].insert(row)
817
      end
818
    end
819
    @db.run "DROP TABLE old_vm_pool;"
820

    
821
    log_time()
822

    
823
    ############################################################################
824
    # Bug #4376 - VLAN IDs Bitmap
825
    ############################################################################
826

    
827
    ## Create and bootstrap 'vlan_bitmap' table
828

    
829
    # Create Table
830
    @db.run "CREATE TABLE network_vlan_bitmap (id INTEGER, map LONGTEXT, PRIMARY KEY(id));"
831

    
832
    size = 4096
833

    
834
    map = ""
835
    size.times.each do |i|
836
        map << (reserved_vlan_ids.include?(size - 1 - i) ? "1" : "0")
837
    end
838

    
839
    map_encoded = Base64::strict_encode64(Zlib::Deflate.deflate(map))
840

    
841
    @db[:network_vlan_bitmap].insert(
842
      :id      => 0,
843
      :map     => map_encoded
844
    )
845

    
846
    log_time()
847

    
848
    ############################################################################
849
    # VNC Bitmap
850
    ############################################################################
851

    
852
    cluster_vnc = {}
853
    @db.transaction do
854
      @db.fetch("SELECT * FROM vm_pool WHERE state != 6") do |row|
855
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
856

    
857
        port = doc.root.at_xpath('TEMPLATE/GRAPHICS[translate(TYPE,"vnc","VNC")="VNC"]/PORT').text.to_i rescue nil
858
        cluster_id = doc.root.at_xpath('HISTORY_RECORDS/HISTORY[last()]/CID').text.to_i rescue nil
859

    
860
        # skip if no port is defined or if it's not assigned to a cluster (not deployed yet!)
861
        next if cluster_id.nil? || port.nil?
862

    
863
        cluster_id = 0 if cluster_id == -1
864

    
865
        cluster_vnc[cluster_id] ||= Set.new
866
        cluster_vnc[cluster_id] << port
867
      end
868
    end
869

    
870
    # Create Table
871
    @db.run "CREATE TABLE cluster_vnc_bitmap (id INTEGER, map LONGTEXT, PRIMARY KEY(id));"
872

    
873
    vnc_pool_size = 65536
874

    
875
    @db.transaction do
876
      @db.fetch("SELECT * FROM cluster_pool") do |row|
877
        cluster_id = row[:oid]
878

    
879
        if cluster_vnc[cluster_id]
880
          map = ""
881
          vnc_pool_size.times.each do |i|
882
            map << (cluster_vnc[cluster_id].include?(vnc_pool_size - 1 - i) ? "1" : "0")
883
          end
884

    
885
          map_encoded = Base64::strict_encode64(Zlib::Deflate.deflate(map))
886
        else
887
          map_encoded = "eJztwYEAAAAAgCCl/ekWqQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABqFo8C0Q=="
888
        end
889

    
890
        @db[:cluster_vnc_bitmap].insert(
891
          :id  => cluster_id,
892
          :map => map_encoded
893
        )
894
      end
895
    end
896

    
897
    log_time()
898

    
899
    ############################################################################
900
    # Feature #4316 - Remove BASE_PATH
901
    ############################################################################
902

    
903
    conf_datastore_location = File.read(File.join(VAR_LOCATION, 'config')).match(/DATASTORE_LOCATION=(.*)$/)[1] rescue nil
904

    
905
    @db.run "ALTER TABLE datastore_pool RENAME TO old_datastore_pool;"
906
    @db.run "CREATE TABLE datastore_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);"
907

    
908
    has_lvm = false
909
    @db.transaction do
910
      @db.fetch("SELECT * FROM old_datastore_pool") do |row|
911
        doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
912

    
913
        ds_id   = row[:oid]
914
        ds_name = row[:name]
915

    
916
        base_path = doc.root.at_xpath("TEMPLATE/BASE_PATH").remove rescue nil
917

    
918
        if base_path
919
          base_path = base_path.text
920

    
921
          if Pathname(base_path).cleanpath != Pathname(conf_datastore_location).cleanpath
922
            puts "**************************************************************"
923
            puts "*  WARNING  WARNING WARNING WARNING WARNING WARNING WARNING  *"
924
            puts "**************************************************************"
925
            puts
926
            puts "The Datastore Attribute BASE_PATH has been deprecated. It has been removed from"
927
            puts "the Datastore template."
928
            puts
929
            puts "We have detected that for the datastore ##{ds_id} (#{ds_name}), it does not match"
930
            puts "the global DATASTORE_LOCATION."
931
            puts
932
            puts "You **MUST** create a symbolic link in the nodes to link them:"
933
            puts "$ ln -s #{base_path} #{conf_datastore_location}"
934
            puts
935
          end
936

    
937
          row[:body] = doc.root.to_s
938
        end
939

    
940
        ds_mad = doc.root.at_xpath("DS_MAD").text rescue nil
941
        has_lvm = true if ds_mad.upcase == "LVM"
942

    
943
        @db[:datastore_pool].insert(row)
944
      end
945
    end
946

    
947
    @db.run "DROP TABLE old_datastore_pool;"
948

    
949
    if has_lvm
950
      puts "**************************************************************"
951
      puts "*  WARNING  WARNING WARNING WARNING WARNING WARNING WARNING  *"
952
      puts "**************************************************************"
953
      puts
954
      puts "The LVM driver is no longer included in the core distribution. It is"
955
      puts "however available as an addon which must be manually installed:"
956
      puts "https://github.com/OpenNebula/addon-lvm"
957
      puts
958
      puts "You have LVM datastores which will not work until you install the"
959
      puts "add-on."
960
      puts
961
      puts "Note that OpenNebula officially recommends using the fs_lvm drivers:"
962
      puts "http://docs.opennebula.org/5.0/deployment/open_cloud_storage_setup/lvm_drivers.html"
963
      puts
964
    end
965

    
966
    log_time()
967

    
968
    return true
969
  end
970

    
971
end