support/cloud-image-builder/customize_cloud_image.py
8ff5c437
 #!/usr/bin/python2
 
 import os
 import re
 import shutil
 import tarfile
 import fileinput
 from optparse import OptionParser
 from utils import Utils
64ead533
 import json
8ff5c437
 
 def create_ova_image(raw_image_name, tools_path, build_scripts_path, config):
     output_path = os.path.dirname(os.path.realpath(raw_image_name))
     utils = Utils()
     # Remove older artifacts
64ead533
     files = os.listdir(output_path)
8ff5c437
     for file in files:
e7b64224
         if file.endswith(".vmdk"):
8ff5c437
             os.remove(os.path.join(output_path, file))
 
     vmx_path = output_path + '/photon-ova.vmx'
     utils.replaceandsaveasnewfile(build_scripts_path + '/vmx-template', vmx_path, 'VMDK_IMAGE', output_path + '/photon-ova.vmdk')
     vixdiskutil_path = tools_path + 'vixdiskutil'
     vmdk_path = output_path + '/photon-ova.vmdk'
     ovf_path = output_path + '/photon-ova.ovf'
     mf_path = output_path + '/photon-ova.mf'
3b85c978
     ovfinfo_path = build_scripts_path + '/ovfinfo.txt'
161db4e9
     vmdk_capacity = (int(config['size']['root']) + int(config['size']['swap'])) * 1024
     utils.runshellcommand("{} -convert {} -cap {} {}".format(vixdiskutil_path, raw_image_name, vmdk_capacity, vmdk_path))
8ff5c437
     utils.runshellcommand("{} -wmeta toolsVersion 2147483647 {}".format(vixdiskutil_path, vmdk_path))
 
     utils.runshellcommand("ovftool {} {}".format(vmx_path, ovf_path))
     utils.replaceinfile(ovf_path, 'otherGuest', 'other3xLinux64Guest')
 
     #Add product info
3b85c978
     if os.path.exists(ovfinfo_path):
         with open(ovfinfo_path) as f:
             lines = f.readlines()
             for line in fileinput.input(ovf_path, inplace=True):
                 if line.strip() == '</VirtualSystem>':
                     for ovfinfoline in lines:
                         print ovfinfoline,
                 print line,
8ff5c437
 
     if os.path.exists(mf_path):
         os.remove(mf_path)
 
     cwd = os.getcwd()
     os.chdir(output_path)
     out = utils.runshellcommand("openssl sha1 photon-ova-disk1.vmdk photon-ova.ovf")
     with open(mf_path, "w") as source:
         source.write(out)
     rawsplit = os.path.splitext(raw_image_name)
     ova_name = rawsplit[0] + '.ova'
 
f75a9580
     ovatar = tarfile.open(ova_name, "w", format = tarfile.USTAR_FORMAT)
8ff5c437
     for name in ["photon-ova.ovf", "photon-ova.mf", "photon-ova-disk1.vmdk"]:
         ovatar.add(name, arcname=os.path.basename(name))
     ovatar.close()
64ead533
     os.remove(vmx_path)
8ff5c437
     os.remove(mf_path)
 
     if 'additionalhwversion' in config:
         for addlversion in config['additionalhwversion']:
e7b64224
             new_ovf_path = output_path + "/photon-ova-hw{}.ovf".format(addlversion)
             mf_path = output_path + "/photon-ova-hw{}.mf".format(addlversion)
64ead533
             utils.replaceandsaveasnewfile(ovf_path, new_ovf_path, "vmx-.*<", "vmx-{}<".format(addlversion))
e7b64224
             out = utils.runshellcommand("openssl sha1 photon-ova-disk1.vmdk photon-ova-hw{}.ovf".format(addlversion))
8ff5c437
             with open(mf_path, "w") as source:
                 source.write(out)
             temp_name_list = os.path.basename(ova_name).split('-')
             temp_name_list = temp_name_list[:2] + ["hw{}".format(addlversion)] + temp_name_list[2:]
             new_ova_name = '-'.join(temp_name_list)
e7b64224
             new_ova_path = output_path + '/' + new_ova_name
f75a9580
             ovatar = tarfile.open(new_ova_path, "w", format = tarfile.USTAR_FORMAT)
8ff5c437
             for name in [new_ovf_path, mf_path, "photon-ova-disk1.vmdk"]:
                 ovatar.add(name, arcname=os.path.basename(name))
             ovatar.close()
 
             os.remove(new_ovf_path)
             os.remove(mf_path)
64ead533
     os.chdir(cwd)
8ff5c437
     os.remove(ovf_path)
     os.remove(vmdk_path)
64ead533
     files = os.listdir(output_path)
e7b64224
     for file in files:
         if file.endswith(".vmdk"):
64ead533
             os.remove(os.path.join(output_path, file))
8ff5c437
 
 
 if __name__ == '__main__':
     usage = "Usage: %prog [options]"
     parser = OptionParser(usage)
 
     parser.add_option("-r", "--raw-image-path",  dest="raw_image_path")
     parser.add_option("-c", "--vmdk-config-path", dest="vmdk_config_path")
     parser.add_option("-w",  "--working-directory",  dest="working_directory")
     parser.add_option("-m",  "--mount-path",  dest="mount_path")
     parser.add_option("-a",  "--additional-rpms-path",  dest="additional_rpms_path")
     parser.add_option("-i",  "--image-name",  dest="image_name")
     parser.add_option("-t",  "--tools-bin-path",  dest="tools_bin_path")
     parser.add_option("-b",  "--build-scripts-path",  dest="build_scripts_path")
94577239
     parser.add_option("-s",  "--src-root",  dest="src_root")
8ff5c437
 
     (options,  args) = parser.parse_args()
     utils = Utils()
     config = utils.jsonread(options.vmdk_config_path)
     print options
 
     disk_device = (utils.runshellcommand("losetup --show -f {}".format(options.raw_image_path))).rstrip('\n')
     disk_partitions = utils.runshellcommand("kpartx -as {}".format(disk_device))
     device_name = disk_device.split('/')[2]
 
     if not os.path.exists(options.mount_path):
         os.mkdir(options.mount_path)
     loop_device_path =  "/dev/mapper/{}p2".format(device_name)
 
     try:
         print "Generating PARTUUID for the loop device ..."
         partuuidval = (utils.runshellcommand("blkid -s PARTUUID -o value {}".format(loop_device_path))).rstrip('\n')
d60ce58d
         uuidval = (utils.runshellcommand("blkid -s UUID -o value {}".format(loop_device_path))).rstrip('\n')
8ff5c437
         if (partuuidval == ''):
             sgdiskout = utils.runshellcommand("sgdisk -i 2 {} ".format(disk_device))
49cc4c52
             partuuidval = (re.findall(r'Partition unique GUID.*', sgdiskout))[0].split(':')[1].strip(' ').lower()
8ff5c437
 
         if (partuuidval == ''):
             raise RuntimeError("Cannot generate partuuid")
 
         # Mount the loop device
         print "Mounting the loop device for customization ..."
         utils.runshellcommand("mount -t ext4 {} {}".format(loop_device_path, options.mount_path))
         shutil.rmtree(options.mount_path + "/installer", ignore_errors=True)
         shutil.rmtree(options.mount_path + "/LOGS", ignore_errors=True)
         # Clear the root password if not set explicitly from the config file
         if (config['password']['text'] != 'PASSWORD'):
             utils.replaceinfile(options.mount_path + "/etc/shadow",'root:.*?:','root:*:')
         # Clear machine-id so it gets regenerated on boot
         open(options.mount_path + "/etc/machine-id", "w").close()
         os.remove(options.mount_path + "/etc/fstab")
 
         f = open(options.mount_path + "/etc/fstab", "w")
d60ce58d
         if (uuidval != ''):
             f.write("UUID={}    /    ext4    defaults 1 1\n".format(uuidval))
         else:
             f.write("PARTUUID={}    /    ext4    defaults 1 1\n".format(partuuidval))
8ff5c437
         f.close()
         utils.replaceinfile(options.mount_path + "/boot/grub/grub.cfg", "rootpartition=PARTUUID=.*$", "rootpartition=PARTUUID={}".format(partuuidval))
 
         if os.path.exists(options.additional_rpms_path):
             print "Installing additional rpms"
             os.mkdir(options.mount_path + "/additional_rpms")
             os.mkdir(options.mount_path + "/var/run")
             utils.copyallfiles(additional_rpms_path, options.mount_path + "/additional_rpms")
             utils.runshellcommand("chroot {} /bin/bash -c 'rpm -i /additional_rpms/*'".format(options.mount_path))
             shutil.rmtree(options.mount_path + "/additional_rpms", ignore_errors=True)
             shutil.rmtree(additional_rpms_path, ignore_errors=True)
 
         utils.runshellcommand("mount -o bind /proc {}".format(options.mount_path + "/proc"))
6eae06bc
         utils.runshellcommand("mount -o rbind /dev {}".format(options.mount_path + "/dev"))
         utils.runshellcommand("mount --make-rslave {}".format(options.mount_path + "/dev"))
8ff5c437
         utils.runshellcommand("mount -o bind /sys {}".format(options.mount_path + "/sys"))
 
         if 'additionalfiles' in config:
c061ff83
             print "  Copying additional files into the raw image ..."
8ff5c437
             for filetuples in config['additionalfiles']:
                 for src,dest in filetuples.iteritems():
                     shutil.copyfile(options.build_scripts_path + '/' + options.image_name + '/' + src, options.mount_path + '/' + dest)
 
 
         if 'postinstallscripts' in config:
c061ff83
             print "  Running post install scripts ..."
8ff5c437
             if not os.path.exists(options.mount_path + "/tempscripts"):
                 os.mkdir(options.mount_path + "/tempscripts")
             for script in config['postinstallscripts']:
                 shutil.copy(options.build_scripts_path + '/' + options.image_name + '/' + script, options.mount_path + "/tempscripts")
             for script in os.listdir(options.mount_path + "/tempscripts"):
                 print "     ...running script {}".format(script)
                 utils.runshellcommand("chroot {} /bin/bash -c '/tempscripts/{}'".format(options.mount_path, script))
             shutil.rmtree(options.mount_path + "/tempscripts", ignore_errors=True)
 
         utils.runshellcommand("umount -l {}".format(options.mount_path + "/sys"))
6eae06bc
         utils.runshellcommand("umount -R -l {}".format(options.mount_path + "/dev"))
8ff5c437
         utils.runshellcommand("umount -l {}".format(options.mount_path + "/proc"))
         utils.runshellcommand("umount -l {}".format(options.mount_path))
 
c061ff83
         mount_out = utils.runshellcommand("mount")
         print "List of mounted devices:"
         print mount_out
 
8ff5c437
     finally:
         utils.runshellcommand("kpartx -d {}".format(disk_device))
         utils.runshellcommand("losetup -d {}".format(disk_device))
 
         shutil.rmtree(options.mount_path)
 
         photon_release_ver = os.environ['PHOTON_RELEASE_VER']
         photon_build_num = os.environ['PHOTON_BUILD_NUM']
         raw_image = options.raw_image_path
         new_name = ""
         img_path = os.path.dirname(os.path.realpath(raw_image))
         # Rename gce image to disk.raw
         if options.image_name == "gce":
             print "Renaming the raw file to disk.raw ..."
             new_name = img_path + '/disk.raw'
 
         else:
             new_name = img_path + '/photon-' + options.image_name + '-' + photon_release_ver + '-' + photon_build_num + '.raw'
 
         shutil.move(raw_image, new_name)
         raw_image = new_name
 
         if config['artifacttype'] == 'tgz':
             print "Generating the tar.gz artifact ..."
             tarname = img_path + '/photon-' + options.image_name + '-' + photon_release_ver + '-' + photon_build_num + '.tar.gz'
c061ff83
             tgzout = tarfile.open(tarname, "w:gz")
8ff5c437
             tgzout.add(raw_image, arcname=os.path.basename(raw_image))
             tgzout.close()
         elif config['artifacttype'] == 'vhd':
94577239
             relrawpath = os.path.relpath(raw_image, options.src_root)
             vhdname = os.path.dirname(relrawpath) + '/photon-' + options.image_name + '-' + photon_release_ver + '-' + photon_build_num + '.vhd'
c061ff83
             print "Converting raw disk to vhd ..."
94577239
             info_output=utils.runshellcommand("docker run -v {}:/mnt:rw anishs/qemu-img info -f raw --output json {}".format(options.src_root, '/mnt/' + relrawpath))
64ead533
             mbsize = 1024 * 1024
b47102eb
             mbroundedsize = ((int(json.loads(info_output)["virtual-size"])/mbsize + 1) * mbsize)
94577239
             utils.runshellcommand("docker run -v {}:/mnt:rw anishs/qemu-img resize -f raw {} {}".format(options.src_root, '/mnt/' + relrawpath, mbroundedsize))
             utils.runshellcommand("docker run -v {}:/mnt:rw anishs/qemu-img convert {} -O vpc -o subformat=fixed,force_size {}".format(options.src_root, '/mnt/' + relrawpath, '/mnt/' + vhdname))
8ff5c437
         elif config['artifacttype'] == 'ova':
             create_ova_image(raw_image, options.tools_bin_path, options.build_scripts_path + '/' + options.image_name, config)
             if 'customartifacts' in config:
                 if 'postinstallscripts' in config['customartifacts']:
                     custom_path = img_path + '/photon-custom'
                     if not os.path.exists(custom_path):
                         os.mkdir(custom_path)
                     index = 1
                     for script in config['customartifacts']['postinstallscripts']:
                         print "Creating custom ova {}...".format(index)
                         if index > 1:
                             raw_image_custom = img_path + "/photon-custom-{}".format(index) + photon_release_ver + '-' + photon_build_num + '.raw'
                         else:
                             raw_image_custom = img_path + "/photon-custom-" + photon_release_ver + '-' + photon_build_num + '.raw'
                         shutil.move(raw_image, raw_image_custom)
                         disk_device = (utils.runshellcommand("losetup --show -f {}".format(raw_image_custom))).rstrip('\n')
                         disk_partitions = utils.runshellcommand("kpartx -as {}".format(disk_device))
                         device_name = disk_device.split('/')[2]
                         loop_device_path =  "/dev/mapper/{}p2".format(device_name)
 
                         print "Mounting the loop device for ova customization ..."
                         utils.runshellcommand("mount -t ext4 {} {}".format(loop_device_path, custom_path))
                         if not os.path.exists(custom_path + "/tempscripts"):
                             os.mkdir(custom_path + "/tempscripts")
                         shutil.copy(options.build_scripts_path + '/' + options.image_name + '/' + script, custom_path + "/tempscripts")
                         print "Running custom ova script {}".format(script)
                         utils.runshellcommand("chroot {} /bin/bash -c '/tempscripts/{}'".format(custom_path, script))
                         shutil.rmtree(custom_path + "/tempscripts", ignore_errors=True)
                         utils.runshellcommand("umount -l {}".format(custom_path))
c061ff83
 
                         mount_out = utils.runshellcommand("mount")
                         print "List of mounted devices:"
                         print mount_out
 
8ff5c437
                         utils.runshellcommand("kpartx -d {}".format(disk_device))
                         utils.runshellcommand("losetup -d {}".format(disk_device))
                         create_ova_image(raw_image_custom, options.tools_bin_path, options.build_scripts_path + '/' + options.image_name, config)
                         raw_image = raw_image_custom
                         index = index + 1
 
64ead533
                     shutil.rmtree(custom_path)
8ff5c437
 
         else:
             raise ValueError("Unknown output format")
 
         if config['keeprawdisk'] == 'false':
49cc4c52
             os.remove(raw_image)