"""
Photon installer
"""
#
# Author: Mahmoud Bassiouny <mbassiouny@vmware.com>
import subprocess
import os
import re
import shutil
import signal
import sys
import glob
import modules.commons
import random
import curses
import stat
import tempfile
from logger import Logger
from commandutils import CommandUtils
from jsonwrapper import JsonWrapper
from progressbar import ProgressBar
from window import Window
from actionresult import ActionResult
from networkmanager import NetworkManager
from enum import Enum
BIOSSIZE = 4
ESPSIZE = 10
class PartitionType(Enum):
SWAP = 1
LINUX = 2
LVM = 3
ESP = 4
BIOS = 5
class Installer(object):
"""
Photon installer
"""
# List of allowed keys in kickstart config file.
# Please keep ks_config.txt file updated.
known_keys = {
'additional_files',
'additional_packages',
'additional_rpms_path',
'arch',
'autopartition',
'bootmode',
'disk',
'eject_cdrom',
'hostname',
'install_linux_esx',
'linux_flavor',
'live',
'log_level',
'ostree',
'packages',
'packagelist_file',
'partition_type',
'partitions',
'network',
'password',
'postinstall',
'postinstallscripts',
'public_key',
'search_path',
'setup_grub_script',
'shadow_password',
'type',
'ui'
}
default_partitions = [{"mountpoint": "/", "size": 0, "filesystem": "ext4"}]
all_linux_flavors = ["linux", "linux-esx", "linux-aws", "linux-secure", "linux-rt"]
linux_dependencies = ["devel", "drivers", "docs", "oprofile", "dtb", "hmacgen"]
def __init__(self, working_directory="/mnt/photon-root",
rpm_path=os.path.dirname(__file__)+"/../stage/RPMS", log_path=os.path.dirname(__file__)+"/../stage/LOGS"):
self.exiting = False
self.interactive = False
self.install_config = None
self.rpm_path = rpm_path
self.log_path = log_path
self.logger = None
self.cmd = None
self.working_directory = working_directory
if os.path.exists(self.working_directory) and os.path.isdir(self.working_directory) and working_directory == '/mnt/photon-root':
shutil.rmtree(self.working_directory)
if not os.path.exists(self.working_directory):
os.mkdir(self.working_directory)
self.photon_root = self.working_directory + "/photon-chroot"
self.installer_path = os.path.dirname(os.path.abspath(__file__))
self.tdnf_conf_path = self.working_directory + "/tdnf.conf"
self.tdnf_repo_path = self.working_directory + "/photon-local.repo"
self.rpm_cache_dir = self.photon_root + '/cache/tdnf/photon-local/rpms'
# used by tdnf.conf as cachedir=, tdnf will append the rest
self.rpm_cache_dir_short = self.photon_root + '/cache/tdnf'
self.setup_grub_command = os.path.dirname(__file__)+"/mk-setup-grub.sh"
signal.signal(signal.SIGINT, self.exit_gracefully)
self.lvs_to_detach = {'vgs': [], 'pvs': []}
"""
create, append and validate configuration date - install_config
"""
def configure(self, install_config, ui_config = None):
# Initialize logger and cmd first
if not install_config:
# UI installation
log_level = 'debug'
console = False
else:
log_level = install_config.get('log_level', 'info')
console = not install_config.get('ui', False)
self.logger = Logger.get_logger(self.log_path, log_level, console)
self.cmd = CommandUtils(self.logger)
# run UI configurator iff install_config param is None
if not install_config and ui_config:
from iso_config import IsoConfig
self.interactive = True
config = IsoConfig()
install_config = curses.wrapper(config.configure, ui_config)
self._add_defaults(install_config)
issue = self._check_install_config(install_config)
if issue:
self.logger.error(issue)
raise Exception(issue)
self.install_config = install_config
def execute(self):
if 'setup_grub_script' in self.install_config:
self.setup_grub_command = self.install_config['setup_grub_script']
if self.install_config['ui']:
curses.wrapper(self._install)
else:
self._install()
def _add_defaults(self, install_config):
"""
Add default install_config settings if not specified
"""
# set arch to host's one if not defined
arch = subprocess.check_output(['uname', '-m'], universal_newlines=True).rstrip('\n')
if 'arch' not in install_config:
install_config['arch'] = arch
# 'bootmode' mode
if 'bootmode' not in install_config:
if "x86_64" in arch:
install_config['bootmode'] = 'dualboot'
else:
install_config['bootmode'] = 'efi'
# extend 'packages' by 'packagelist_file' and 'additional_packages'
packages = []
if 'packagelist_file' in install_config:
plf = install_config['packagelist_file']
if not plf.startswith('/'):
plf = os.path.join(os.path.dirname(__file__), plf)
json_wrapper_package_list = JsonWrapper(plf)
package_list_json = json_wrapper_package_list.read()
if "packages_" + install_config['arch'] in package_list_json:
packages.extend(package_list_json["packages"] + package_list_json["packages_"+install_config['arch']])
else:
packages.extend(package_list_json["packages"])
if 'additional_packages' in install_config:
packages.extend(install_config['additional_packages'])
# add bootloader packages after bootmode set
if install_config['bootmode'] in ['dualboot', 'efi']:
packages.append('grub2-efi-image')
if 'packages' in install_config:
install_config['packages'] = list(set(packages + install_config['packages']))
else:
install_config['packages'] = packages
# live means online system. When you create an image for
# target system, live should be set to false.
if 'live' not in install_config:
install_config['live'] = 'loop' not in install_config['disk']
# default partition
if 'partitions' not in install_config:
install_config['partitions'] = Installer.default_partitions
# define 'hostname' as 'photon-<RANDOM STRING>'
if "hostname" not in install_config or install_config['hostname'] == "":
install_config['hostname'] = 'photon-%12x' % random.randrange(16**12)
# Set password if needed.
# Installer uses 'shadow_password' and optionally 'password'/'age'
# to set aging if present. See modules/m_updaterootpassword.py
if 'shadow_password' not in install_config:
if 'password' not in install_config:
install_config['password'] = {'crypted': True, 'text': '*', 'age': -1}
if install_config['password']['crypted']:
install_config['shadow_password'] = install_config['password']['text']
else:
install_config['shadow_password'] = CommandUtils.generate_password_hash(install_config['password']['text'])
# Do not show UI progress by default
if 'ui' not in install_config:
install_config['ui'] = False
# Log level
if 'log_level' not in install_config:
install_config['log_level'] = 'info'
# Extend search_path by current dir and script dir
if 'search_path' not in install_config:
install_config['search_path'] = []
for dirname in [os.getcwd(), os.path.abspath(os.path.dirname(__file__))]:
if dirname not in install_config['search_path']:
install_config['search_path'].append(dirname)
if 'linux_flavor' not in install_config:
if install_config.get('install_linux_esx', False) == True:
install_config['linux_flavor'] = "linux-esx"
else:
available_flavors=[]
for flavor in self.all_linux_flavors:
if flavor in install_config['packages']:
available_flavors.append(flavor)
if len(available_flavors) == 1:
install_config['linux_flavor'] = available_flavors[0]
install_config['install_linux_esx'] = False
def _check_install_config(self, install_config):
"""
Sanity check of install_config before its execution.
Return error string or None
"""
unknown_keys = install_config.keys() - Installer.known_keys
if len(unknown_keys) > 0:
return "Unknown install_config keys: " + ", ".join(unknown_keys)
if not 'disk' in install_config:
return "No disk configured"
# For Ostree install_config['packages'] will be empty list, because ostree
# uses preinstalled tree ostree-repo.tar.gz for installation
if 'ostree' not in install_config and 'linux_flavor' not in install_config:
return "Attempting to install more than one linux flavor"
# Perform following checks here:
# 1) Only one extensible partition is allowed per disk
# 2) /boot can not be LVM
# 3) / must present
# 4) Duplicate mountpoints should not be present
has_extensible = {}
has_root = False
mountpoints = []
default_disk = install_config['disk']
for partition in install_config['partitions']:
disk = partition.get('disk', default_disk)
mntpoint = partition.get('mountpoint', '')
if disk not in has_extensible:
has_extensible[disk] = False
size = partition['size']
if size == 0:
if has_extensible[disk]:
return "Disk {} has more than one extensible partition".format(disk)
else:
has_extensible[disk] = True
if mntpoint != '':
mountpoints.append(mntpoint)
if mntpoint == '/boot' and 'lvm' in partition:
return "/boot on LVM is not supported"
elif mntpoint == '/':
has_root = True
if not has_root:
return "There is no partition assigned to root '/'"
if len(mountpoints) != len(set(mountpoints)):
return "Duplicate mountpoints exist in partition table!!"
if install_config['arch'] not in ["aarch64", 'x86_64']:
return "Unsupported target architecture {}".format(install_config['arch'])
# No BIOS for aarch64
if install_config['arch'] == 'aarch64' and install_config['bootmode'] in ['dualboot', 'bios']:
return "Aarch64 targets do not support BIOS boot. Set 'bootmode' to 'efi'."
if 'age' in install_config.get('password', {}):
if install_config['password']['age'] < -1:
return "Password age should be -1, 0 or positive"
return None
def _install(self, stdscreen=None):
"""
Install photon system and handle exception
"""
if self.install_config['ui']:
# init the screen
curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE)
curses.init_pair(2, curses.COLOR_BLACK, curses.COLOR_WHITE)
curses.init_pair(3, curses.COLOR_BLACK, curses.COLOR_GREEN)
curses.init_pair(4, curses.COLOR_RED, curses.COLOR_WHITE)
stdscreen.bkgd(' ', curses.color_pair(1))
maxy, maxx = stdscreen.getmaxyx()
curses.curs_set(0)
# initializing windows
height = 10
width = 75
progress_padding = 5
progress_width = width - progress_padding
starty = (maxy - height) // 2
startx = (maxx - width) // 2
self.window = Window(height, width, maxy, maxx,
'Installing Photon', False)
self.progress_bar = ProgressBar(starty + 3,
startx + progress_padding // 2,
progress_width)
self.window.show_window()
self.progress_bar.initialize('Initializing installation...')
self.progress_bar.show()
try:
self._unsafe_install()
except Exception as inst:
self.logger.exception(repr(inst))
self.exit_gracefully()
# Congratulation screen
if self.install_config['ui']:
self.progress_bar.hide()
self.window.addstr(0, 0, 'Congratulations, Photon has been installed in {0} secs.\n\n'
'Press any key to continue to boot...'
.format(self.progress_bar.time_elapsed))
if self.interactive:
self.window.content_window().getch()
if self.install_config['live']:
self._eject_cdrom()
def _unsafe_install(self):
"""
Install photon system
"""
self._partition_disk()
self._format_partitions()
self._mount_partitions()
if 'ostree' in self.install_config:
from ostreeinstaller import OstreeInstaller
ostree = OstreeInstaller(self)
ostree.install()
else:
self._setup_install_repo()
self._initialize_system()
self._mount_special_folders()
self._install_packages()
self._install_additional_rpms()
self._enable_network_in_chroot()
self._setup_network()
self._finalize_system()
self._cleanup_install_repo()
self._setup_grub()
self._create_fstab()
self._execute_modules(modules.commons.POST_INSTALL)
self._disable_network_in_chroot()
self._unmount_all()
def exit_gracefully(self, signal1=None, frame1=None):
"""
This will be called if the installer interrupted by Ctrl+C, exception
or other failures
"""
del signal1
del frame1
if not self.exiting and self.install_config:
self.exiting = True
if self.install_config['ui']:
self.progress_bar.hide()
self.window.addstr(0, 0, 'Oops, Installer got interrupted.\n\n' +
'Press any key to get to the bash...')
self.window.content_window().getch()
self._cleanup_install_repo()
self._unmount_all()
sys.exit(1)
def _setup_network(self):
if 'network' not in self.install_config:
return
# setup network config files in chroot
nm = NetworkManager(self.install_config, self.photon_root)
if not nm.setup_network():
self.logger.error("Failed to setup network!")
self.exit_gracefully()
# Configure network when in live mode (ISO) and when network is not
# already configured (typically in KS flow).
if ('live' in self.install_config and
'conf_files' not in self.install_config['network']):
nm = NetworkManager(self.install_config)
if not nm.setup_network():
self.logger.error("Failed to setup network in ISO system")
self.exit_gracefully()
nm.restart_networkd()
def _unmount_all(self):
"""
Unmount partitions and special folders
"""
for d in ["/tmp", "/run", "/sys", "/dev/pts", "/dev", "/proc"]:
if os.path.exists(self.photon_root + d):
retval = self.cmd.run(['umount', '-l', self.photon_root + d])
if retval != 0:
self.logger.error("Failed to unmount {}".format(d))
for partition in self.install_config['partitions'][::-1]:
if self._get_partition_type(partition) in [PartitionType.BIOS, PartitionType.SWAP]:
continue
mountpoint = self.photon_root + partition["mountpoint"]
if os.path.exists(mountpoint):
retval = self.cmd.run(['umount', '-l', mountpoint])
if retval != 0:
self.logger.error("Failed to unmount partition {}".format(mountpoint))
# need to call it twice, because of internal bind mounts
if 'ostree' in self.install_config:
if os.path.exists(self.photon_root):
retval = self.cmd.run(['umount', '-R', self.photon_root])
retval = self.cmd.run(['umount', '-R', self.photon_root])
if retval != 0:
self.logger.error("Failed to unmount disks in photon root")
self.cmd.run(['sync'])
if os.path.exists(self.photon_root):
shutil.rmtree(self.photon_root)
# Deactivate LVM VGs
for vg in self.lvs_to_detach['vgs']:
retval = self.cmd.run(["vgchange", "-v", "-an", vg])
if retval != 0:
self.logger.error("Failed to deactivate LVM volume group: {}".format(vg))
# Get the disks from partition table
disks = set(partition.get('disk', self.install_config['disk']) for partition in self.install_config['partitions'])
for disk in disks:
if 'loop' in disk:
# Simulate partition hot remove to notify LVM
for pv in self.lvs_to_detach['pvs']:
retval = self.cmd.run(["dmsetup", "remove", pv])
if retval != 0:
self.logger.error("Failed to detach LVM physical volume: {}".format(pv))
# Uninitialize device paritions mapping
retval = self.cmd.run(['kpartx', '-d', disk])
if retval != 0:
self.logger.error("Failed to unmap partitions of the disk image {}". format(disk))
return None
def _bind_installer(self):
"""
Make the photon_root/installer directory if not exits
The function finalize_system will access the file /installer/mk-finalize-system.sh
after chroot to photon_root.
Bind the /installer folder to self.photon_root/installer, so that after chroot
to photon_root,
the file can still be accessed as /installer/mk-finalize-system.sh.
"""
# Make the photon_root/installer directory if not exits
if(self.cmd.run(['mkdir', '-p',
os.path.join(self.photon_root, "installer")]) != 0 or
self.cmd.run(['mount', '--bind', self.installer_path,
os.path.join(self.photon_root, "installer")]) != 0):
self.logger.error("Fail to bind installer")
self.exit_gracefully()
def _unbind_installer(self):
# unmount the installer directory
if os.path.exists(os.path.join(self.photon_root, "installer")):
retval = self.cmd.run(['umount', os.path.join(self.photon_root, "installer")])
if retval != 0:
self.logger.error("Fail to unbind the installer directory")
# remove the installer directory
retval = self.cmd.run(['rm', '-rf', os.path.join(self.photon_root, "installer")])
if retval != 0:
self.logger.error("Fail to remove the installer directory")
def _bind_repo_dir(self):
"""
Bind repo dir for tdnf installation
"""
if self.rpm_path.startswith("https://") or self.rpm_path.startswith("http://"):
return
if (self.cmd.run(['mkdir', '-p', self.rpm_cache_dir]) != 0 or
self.cmd.run(['mount', '--bind', self.rpm_path, self.rpm_cache_dir]) != 0):
self.logger.error("Fail to bind cache rpms")
self.exit_gracefully()
def _unbind_repo_dir(self):
"""
Unbind repo dir after installation
"""
if self.rpm_path.startswith("https://") or self.rpm_path.startswith("http://"):
return
if os.path.exists(self.rpm_cache_dir):
if (self.cmd.run(['umount', self.rpm_cache_dir]) != 0 or
self.cmd.run(['rm', '-rf', self.rpm_cache_dir]) != 0):
self.logger.error("Fail to unbind cache rpms")
def _get_partuuid(self, path):
partuuid = subprocess.check_output(['blkid', '-s', 'PARTUUID', '-o', 'value', path],
universal_newlines=True).rstrip('\n')
# Backup way to get uuid/partuuid. Leave it here for later use.
#if partuuidval == '':
# sgdiskout = Utils.runshellcommand(
# "sgdisk -i 2 {} ".format(disk_device))
# partuuidval = (re.findall(r'Partition unique GUID.*',
# sgdiskout))[0].split(':')[1].strip(' ').lower()
return partuuid
def _get_uuid(self, path):
return subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', path],
universal_newlines=True).rstrip('\n')
def _create_fstab(self, fstab_path = None):
"""
update fstab
"""
if not fstab_path:
fstab_path = os.path.join(self.photon_root, "etc/fstab")
with open(fstab_path, "w") as fstab_file:
fstab_file.write("#system\tmnt-pt\ttype\toptions\tdump\tfsck\n")
for partition in self.install_config['partitions']:
ptype = self._get_partition_type(partition)
if ptype == PartitionType.BIOS:
continue
options = 'defaults'
dump = 1
fsck = 2
if partition.get('mountpoint', '') == '/':
options = options + ',barrier,noatime,noacl,data=ordered'
fsck = 1
if ptype == PartitionType.SWAP:
mountpoint = 'swap'
dump = 0
fsck = 0
else:
mountpoint = partition['mountpoint']
# Use PARTUUID/UUID instead of bare path.
# Prefer PARTUUID over UUID as it is supported by kernel
# and UUID only by initrd.
path = partition['path']
mnt_src = None
partuuid = self._get_partuuid(path)
if partuuid != '':
mnt_src = "PARTUUID={}".format(partuuid)
else:
uuid = self._get_uuid(path)
if uuid != '':
mnt_src = "UUID={}".format(uuid)
if not mnt_src:
raise RuntimeError("Cannot get PARTUUID/UUID of: {}".format(path))
fstab_file.write("{}\t{}\t{}\t{}\t{}\t{}\n".format(
mnt_src,
mountpoint,
partition['filesystem'],
options,
dump,
fsck
))
# Add the cdrom entry
fstab_file.write("/dev/cdrom\t/mnt/cdrom\tiso9660\tro,noauto\t0\t0\n")
def _generate_partitions_param(self, reverse=False):
"""
Generate partition param for mount command
"""
if reverse:
step = -1
else:
step = 1
params = []
for partition in self.install_config['partitions'][::step]:
if self._get_partition_type(partition) in [PartitionType.BIOS, PartitionType.SWAP]:
continue
params.extend(['--partitionmountpoint', partition["path"], partition["mountpoint"]])
return params
def _mount_partitions(self):
for partition in self.install_config['partitions'][::1]:
if self._get_partition_type(partition) in [PartitionType.BIOS, PartitionType.SWAP]:
continue
mountpoint = self.photon_root + partition["mountpoint"]
self.cmd.run(['mkdir', '-p', mountpoint])
retval = self.cmd.run(['mount', '-v', partition["path"], mountpoint])
if retval != 0:
self.logger.error("Failed to mount partition {}".format(partition["path"]))
self.exit_gracefully()
def _initialize_system(self):
"""
Prepare the system to install photon
"""
if self.install_config['ui']:
self.progress_bar.update_message('Initializing system...')
self._bind_installer()
self._bind_repo_dir()
# Initialize rpm DB
self.cmd.run(['mkdir', '-p', os.path.join(self.photon_root, "var/lib/rpm")])
retval = self.cmd.run(['rpm', '--root', self.photon_root, '--initdb',
'--dbpath', '/var/lib/rpm'])
if retval != 0:
self.logger.error("Failed to initialize rpm DB")
self.exit_gracefully()
# Install filesystem rpm
tdnf_cmd = "tdnf install filesystem --installroot {0} --assumeyes -c {1}".format(self.photon_root,
self.tdnf_conf_path)
retval = self.cmd.run(tdnf_cmd)
if retval != 0:
retval = self.cmd.run(['docker', 'run',
'-v', self.rpm_cache_dir+':'+self.rpm_cache_dir,
'-v', self.working_directory+':'+self.working_directory,
'photon:3.0', '/bin/sh', '-c', tdnf_cmd])
if retval != 0:
self.logger.error("Failed to install filesystem rpm")
self.exit_gracefully()
# Create special devices. We need it when devtpmfs is not mounted yet.
devices = {
'console': (600, stat.S_IFCHR, 5, 1),
'null': (666, stat.S_IFCHR, 1, 3),
'random': (444, stat.S_IFCHR, 1, 8),
'urandom': (444, stat.S_IFCHR, 1, 9)
}
for device, (mode, dev_type, major, minor) in devices.items():
os.mknod(os.path.join(self.photon_root, "dev", device),
mode | dev_type, os.makedev(major, minor))
def _mount_special_folders(self):
for d in ["/proc", "/dev", "/dev/pts", "/sys"]:
retval = self.cmd.run(['mount', '-o', 'bind', d, self.photon_root + d])
if retval != 0:
self.logger.error("Failed to bind mount {}".format(d))
self.exit_gracefully()
for d in ["/tmp", "/run"]:
retval = self.cmd.run(['mount', '-t', 'tmpfs', 'tmpfs', self.photon_root + d])
if retval != 0:
self.logger.error("Failed to bind mount {}".format(d))
self.exit_gracefully()
def _copy_additional_files(self):
if 'additional_files' in self.install_config:
for filetuples in self.install_config['additional_files']:
for src, dest in filetuples.items():
if src.startswith('http://') or src.startswith('https://'):
temp_file = tempfile.mktemp()
result, msg = CommandUtils.wget(src, temp_file, False)
if result:
shutil.copyfile(temp_file, self.photon_root + dest)
else:
self.logger.error("Download failed URL: {} got error: {}".format(src, msg))
else:
srcpath = self.getfile(src)
if (os.path.isdir(srcpath)):
shutil.copytree(srcpath, self.photon_root + dest, True)
else:
shutil.copyfile(srcpath, self.photon_root + dest)
def _finalize_system(self):
"""
Finalize the system after the installation
"""
if self.install_config['ui']:
self.progress_bar.show_loading('Finalizing installation')
self._copy_additional_files()
self.cmd.run_in_chroot(self.photon_root, "/sbin/ldconfig")
# Importing the pubkey
self.cmd.run_in_chroot(self.photon_root, "rpm --import /etc/pki/rpm-gpg/*")
def _cleanup_install_repo(self):
self._unbind_installer()
self._unbind_repo_dir()
# remove the tdnf cache directory.
retval = self.cmd.run(['rm', '-rf', os.path.join(self.photon_root, "cache")])
if retval != 0:
self.logger.error("Fail to remove the cache")
if os.path.exists(self.tdnf_conf_path):
os.remove(self.tdnf_conf_path)
if os.path.exists(self.tdnf_repo_path):
os.remove(self.tdnf_repo_path)
def _setup_grub(self):
bootmode = self.install_config['bootmode']
# Setup bios grub
if bootmode == 'dualboot' or bootmode == 'bios':
retval = self.cmd.run('grub2-install --target=i386-pc --force --boot-directory={} {}'.format(self.photon_root + "/boot", self.install_config['disk']))
if retval != 0:
retval = self.cmd.run(['grub-install', '--target=i386-pc', '--force',
'--boot-directory={}'.format(self.photon_root + "/boot"),
self.install_config['disk']])
if retval != 0:
raise Exception("Unable to setup grub")
# Setup efi grub
if bootmode == 'dualboot' or bootmode == 'efi':
esp_pn = '1'
if bootmode == 'dualboot':
esp_pn = '2'
self.cmd.run(['mkdir', '-p', self.photon_root + '/boot/efi/boot/grub2'])
with open(os.path.join(self.photon_root, 'boot/efi/boot/grub2/grub.cfg'), "w") as grub_cfg:
grub_cfg.write("search -n -u {} -s\n".format(self._get_uuid(self.install_config['partitions_data']['boot'])))
grub_cfg.write("set prefix=($root){}grub2\n".format(self.install_config['partitions_data']['bootdirectory']))
grub_cfg.write("configfile {}grub2/grub.cfg\n".format(self.install_config['partitions_data']['bootdirectory']))
if self.install_config['live']:
arch = self.install_config['arch']
# 'x86_64' -> 'bootx64.efi', 'aarch64' -> 'bootaa64.efi'
exe_name = 'boot'+arch[:-5]+arch[-2:]+'.efi'
# Some platforms do not support adding boot entry. Thus, ignore failures
self.cmd.run(['efibootmgr', '--create', '--remove-dups', '--disk', self.install_config['disk'],
'--part', esp_pn, '--loader', '/EFI/BOOT/' + exe_name, '--label', 'Photon'])
# Create custom grub.cfg
retval = self.cmd.run(
[self.setup_grub_command, self.photon_root,
self.install_config['partitions_data']['root'],
self.install_config['partitions_data']['boot'],
self.install_config['partitions_data']['bootdirectory']])
if retval != 0:
raise Exception("Bootloader (grub2) setup failed")
def _execute_modules(self, phase):
"""
Execute the scripts in the modules folder
"""
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "modules")))
modules_paths = glob.glob(os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) + '/m_*.py')
for mod_path in modules_paths:
module = os.path.splitext(os.path.basename(mod_path))[0]
try:
__import__(module)
mod = sys.modules[module]
except ImportError:
self.logger.error('Error importing module {}'.format(module))
continue
# the module default is disabled
if not hasattr(mod, 'enabled') or mod.enabled is False:
self.logger.info("module {} is not enabled".format(module))
continue
# check for the install phase
if not hasattr(mod, 'install_phase'):
self.logger.error("Error: can not defind module {} phase".format(module))
continue
if mod.install_phase != phase:
self.logger.info("Skipping module {0} for phase {1}".format(module, phase))
continue
if not hasattr(mod, 'execute'):
self.logger.error("Error: not able to execute module {}".format(module))
continue
self.logger.info("Executing: " + module)
mod.execute(self)
def _adjust_packages_based_on_selected_flavor(self):
"""
Install slected linux flavor only
"""
redundant_linux_flavors = []
def filter_packages(package):
package = package.split('-')
if len(package) > 1:
flavor = package[1]
else:
flavor = ""
if(package[0] != "linux"):
return True
elif("" in redundant_linux_flavors and flavor in self.linux_dependencies):
return False
elif(flavor in redundant_linux_flavors):
return False
else:
return True
for flavor in self.all_linux_flavors:
if(flavor != self.install_config['linux_flavor']):
flavor = flavor.split('-')
if len(flavor) > 1:
flavor = flavor[1]
else:
flavor = ""
redundant_linux_flavors.append(flavor)
self.install_config['packages'] = list(filter(filter_packages,self.install_config['packages']))
def _add_packages_to_install(self, package):
"""
Install packages on Vmware virtual machine if requested
"""
self.install_config['packages'].append(package)
def _setup_install_repo(self):
"""
Setup the tdnf repo for installation
"""
keepcache = False
with open(self.tdnf_repo_path, "w") as repo_file:
repo_file.write("[photon-local]\n")
repo_file.write("name=VMWare Photon installer repo\n")
if self.rpm_path.startswith("https://") or self.rpm_path.startswith("http://"):
repo_file.write("baseurl={}\n".format(self.rpm_path))
else:
repo_file.write("baseurl=file://{}\n".format(self.rpm_cache_dir))
keepcache = True
repo_file.write("gpgcheck=0\nenabled=1\n")
with open(self.tdnf_conf_path, "w") as conf_file:
conf_file.writelines([
"[main]\n",
"gpgcheck=0\n",
"installonly_limit=3\n",
"clean_requirements_on_remove=true\n"])
# baseurl and cachedir are bindmounted to rpm_path, we do not
# want input RPMS to be removed after installation.
if keepcache:
conf_file.write("keepcache=1\n")
conf_file.write("repodir={}\n".format(self.working_directory))
conf_file.write("cachedir={}\n".format(self.rpm_cache_dir_short))
def _install_additional_rpms(self):
rpms_path = self.install_config.get('additional_rpms_path', None)
if not rpms_path or not os.path.exists(rpms_path):
return
if self.cmd.run([ 'rpm', '--root', self.photon_root, '-U', rpms_path + '/*.rpm' ]) != 0:
self.logger.info('Failed to install additional_rpms from ' + rpms_path)
self.exit_gracefully()
def _install_packages(self):
"""
Install packages using tdnf command
"""
self._adjust_packages_based_on_selected_flavor()
selected_packages = self.install_config['packages']
state = 0
packages_to_install = {}
total_size = 0
stderr = None
tdnf_cmd = "tdnf install --installroot {0} --assumeyes -c {1} {2}".format(self.photon_root,
self.tdnf_conf_path, " ".join(selected_packages))
self.logger.debug(tdnf_cmd)
# run in shell to do not throw exception if tdnf not found
process = subprocess.Popen(tdnf_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if self.install_config['ui']:
while True:
output = process.stdout.readline().decode()
if output == '':
retval = process.poll()
if retval is not None:
stderr = process.communicate()[1]
break
if state == 0:
if output == 'Installing:\n':
state = 1
elif state == 1: #N A EVR Size(readable) Size(in bytes)
if output == '\n':
state = 2
self.progress_bar.update_num_items(total_size)
else:
info = output.split()
package = '{0}-{1}.{2}'.format(info[0], info[2], info[1])
packages_to_install[package] = int(info[5])
total_size += int(info[5])
elif state == 2:
if output == 'Downloading:\n':
self.progress_bar.update_message('Preparing ...')
state = 3
elif state == 3:
self.progress_bar.update_message(output)
if output == 'Running transaction\n':
state = 4
else:
self.logger.info("[tdnf] {0}".format(output))
prefix = 'Installing/Updating: '
if output.startswith(prefix):
package = output[len(prefix):].rstrip('\n')
self.progress_bar.increment(packages_to_install[package])
self.progress_bar.update_message(output)
else:
stdout,stderr = process.communicate()
self.logger.info(stdout.decode())
retval = process.returncode
# image creation. host's tdnf might not be available or can be outdated (Photon 1.0)
# retry with docker container
if retval != 0 and retval != 137:
self.logger.error(stderr.decode())
stderr = None
self.logger.info("Retry 'tdnf install' using docker image")
retval = self.cmd.run(['docker', 'run',
'-v', self.rpm_cache_dir+':'+self.rpm_cache_dir,
'-v', self.working_directory+':'+self.working_directory,
'photon:3.0', '/bin/sh', '-c', tdnf_cmd])
# 0 : succeed; 137 : package already installed; 65 : package not found in repo.
if retval != 0 and retval != 137:
self.logger.error("Failed to install some packages")
if stderr:
self.logger.error(stderr.decode())
self.exit_gracefully()
def _eject_cdrom(self):
"""
Eject the cdrom on request
"""
if self.install_config.get('eject_cdrom', True):
self.cmd.run(['eject', '-r'])
def _enable_network_in_chroot(self):
"""
Enable network in chroot
"""
if os.path.exists("/etc/resolv.conf"):
shutil.copy("/etc/resolv.conf", self.photon_root + '/etc/.')
def _disable_network_in_chroot(self):
"""
disable network in chroot
"""
if os.path.exists(self.photon_root + '/etc/resolv.conf'):
os.remove(self.photon_root + '/etc/resolv.conf')
def partition_compare(self, p):
if 'mountpoint' in p:
return (1, len(p['mountpoint']), p['mountpoint'])
return (0, 0, "A")
def _get_partition_path(self, disk, part_idx):
prefix = ''
if 'nvme' in disk or 'mmcblk' in disk or 'loop' in disk:
prefix = 'p'
# loop partitions device names are /dev/mapper/loopXpY instead of /dev/loopXpY
if 'loop' in disk:
path = '/dev/mapper' + disk[4:] + prefix + repr(part_idx)
else:
path = disk + prefix + repr(part_idx)
return path
def _get_partition_type(self, partition):
if partition['filesystem'] == 'bios':
return PartitionType.BIOS
if partition['filesystem'] == 'swap':
return PartitionType.SWAP
if partition.get('mountpoint', '') == '/boot/efi' and partition['filesystem'] == 'vfat':
return PartitionType.ESP
if partition.get('lvm', None):
return PartitionType.LVM
return PartitionType.LINUX
def _partition_type_to_string(self, ptype):
if ptype == PartitionType.BIOS:
return 'ef02'
if ptype == PartitionType.SWAP:
return '8200'
if ptype == PartitionType.ESP:
return 'ef00'
if ptype == PartitionType.LVM:
return '8e00'
if ptype == PartitionType.LINUX:
return '8300'
raise Exception("Unknown partition type: {}".format(ptype))
def _create_logical_volumes(self, physical_partition, vg_name, lv_partitions, extensible):
"""
Create logical volumes
"""
#Remove LVM logical volumes and volume groups if already exists
#Existing lvs & vg should be removed to continue re-installation
#else pvcreate command fails to create physical volumes even if executes forcefully
retval = self.cmd.run(['bash', '-c', 'pvs | grep {}'. format(vg_name)])
if retval == 0:
#Remove LV's associated to VG and VG
retval = self.cmd.run(["vgremove", "-f", vg_name])
if retval != 0:
self.logger.error("Error: Failed to remove existing vg before installation {}". format(vg_name))
# if vg is not extensible (all lvs inside are known size) then make last lv
# extensible, i.e. shrink it. Srinking last partition is important. We will
# not be able to provide specified size because given physical partition is
# also used by LVM header.
extensible_logical_volume = None
if not extensible:
extensible_logical_volume = lv_partitions[-1]
extensible_logical_volume['size'] = 0
# create physical volume
command = ['pvcreate', '-ff', '-y', physical_partition]
retval = self.cmd.run(command)
if retval != 0:
raise Exception("Error: Failed to create physical volume, command : {}".format(command))
# create volume group
command = ['vgcreate', vg_name, physical_partition]
retval = self.cmd.run(command)
if retval != 0:
raise Exception("Error: Failed to create volume group, command = {}".format(command))
# create logical volumes
for partition in lv_partitions:
lv_cmd = ['lvcreate', '-y']
lv_name = partition['lvm']['lv_name']
size = partition['size']
if partition['size'] == 0:
# Each volume group can have only one extensible logical volume
if not extensible_logical_volume:
extensible_logical_volume = partition
else:
lv_cmd.extend(['-L', '{}M'.format(partition['size']), '-n', lv_name, vg_name ])
retval = self.cmd.run(lv_cmd)
if retval != 0:
raise Exception("Error: Failed to create logical volumes , command: {}".format(lv_cmd))
partition['path'] = '/dev/' + vg_name + '/' + lv_name
# create extensible logical volume
if not extensible_logical_volume:
raise Exception("Can not fully partition VG: " + vg_name)
lv_name = extensible_logical_volume['lvm']['lv_name']
lv_cmd = ['lvcreate', '-y']
lv_cmd.extend(['-l', '100%FREE', '-n', lv_name, vg_name ])
retval = self.cmd.run(lv_cmd)
if retval != 0:
raise Exception("Error: Failed to create extensible logical volume, command = {}". format(lv_cmd))
# remember pv/vg for detaching it later.
self.lvs_to_detach['pvs'].append(os.path.basename(physical_partition))
self.lvs_to_detach['vgs'].append(vg_name)
def _get_partition_tree_view(self):
# Tree View of partitions list, to be returned.
# 1st level: dict of disks
# 2nd level: list of physical partitions, with all information necessary to partition the disk
# 3rd level: list of logical partitions (LVM) or detailed partition information needed to format partition
ptv = {}
# Dict of VG's per disk. Purpose of this dict is:
# 1) to collect its LV's
# 2) to accumulate total size
# 3) to create physical partition representation for VG
vg_partitions = {}
default_disk = self.install_config['disk']
partitions = self.install_config['partitions']
for partition in partitions:
disk = partition.get('disk', default_disk)
if disk not in ptv:
ptv[disk] = []
if disk not in vg_partitions:
vg_partitions[disk] = {}
if partition.get('lvm', None):
vg_name = partition['lvm']['vg_name']
if vg_name not in vg_partitions[disk]:
vg_partitions[disk][vg_name] = {
'size': 0,
'type': self._partition_type_to_string(PartitionType.LVM),
'extensible': False,
'lvs': [],
'vg_name': vg_name
}
vg_partitions[disk][vg_name]['lvs'].append(partition)
if partition['size'] == 0:
vg_partitions[disk][vg_name]['extensible'] = True
vg_partitions[disk][vg_name]['size'] = 0
else:
if not vg_partitions[disk][vg_name]['extensible']:
vg_partitions[disk][vg_name]['size'] = vg_partitions[disk][vg_name]['size'] + partition['size']
else:
if 'type' in partition:
ptype_code = partition['type']
else:
ptype_code = self._partition_type_to_string(self._get_partition_type(partition))
l2entry = {
'size': partition['size'],
'type': ptype_code,
'partition': partition
}
ptv[disk].append(l2entry)
# Add accumulated VG partitions
for disk, vg_list in vg_partitions.items():
ptv[disk].extend(vg_list.values())
return ptv
def _insert_boot_partitions(self):
bios_found = False
esp_found = False
for partition in self.install_config['partitions']:
ptype = self._get_partition_type(partition)
if ptype == PartitionType.BIOS:
bios_found = True
if ptype == PartitionType.ESP:
esp_found = True
# Adding boot partition required for ostree if already not present in partitions table
if 'ostree' in self.install_config:
mount_points = [partition['mountpoint'] for partition in self.install_config['partitions'] if 'mountpoint' in partition]
if '/boot' not in mount_points:
boot_partition = {'size': 300, 'filesystem': 'ext4', 'mountpoint': '/boot'}
self.install_config['partitions'].insert(0, boot_partition)
bootmode = self.install_config.get('bootmode', 'bios')
# Insert efi special partition
if not esp_found and (bootmode == 'dualboot' or bootmode == 'efi'):
efi_partition = { 'size': ESPSIZE, 'filesystem': 'vfat', 'mountpoint': '/boot/efi' }
self.install_config['partitions'].insert(0, efi_partition)
# Insert bios partition last to be very first
if not bios_found and (bootmode == 'dualboot' or bootmode == 'bios'):
bios_partition = { 'size': BIOSSIZE, 'filesystem': 'bios' }
self.install_config['partitions'].insert(0, bios_partition)
def _partition_disk(self):
"""
Partition the disk
"""
if self.install_config['ui']:
self.progress_bar.update_message('Partitioning...')
self._insert_boot_partitions()
ptv = self._get_partition_tree_view()
partitions = self.install_config['partitions']
partitions_data = {}
lvm_present = False
# Partitioning disks
for disk, l2entries in ptv.items():
# Clear the disk first
retval = self.cmd.run(['sgdisk', '-o', '-g', disk])
if retval != 0:
raise Exception("Failed clearing disk {0}".format(disk))
# Build partition command and insert 'part' into 'partitions'
partition_cmd = ['sgdisk']
part_idx = 1
# command option for extensible partition
last_partition = None
for l2 in l2entries:
if 'lvs' in l2:
# will be used for _create_logical_volumes() invocation
l2['path'] = self._get_partition_path(disk, part_idx)
else:
l2['partition']['path'] = self._get_partition_path(disk, part_idx)
if l2['size'] == 0:
last_partition = []
last_partition.extend(['-n{}'.format(part_idx)])
last_partition.extend(['-t{}:{}'.format(part_idx, l2['type'])])
else:
partition_cmd.extend(['-n{}::+{}M'.format(part_idx, l2['size'])])
partition_cmd.extend(['-t{}:{}'.format(part_idx, l2['type'])])
part_idx = part_idx + 1
# if extensible partition present, add it to the end of the disk
if last_partition:
partition_cmd.extend(last_partition)
partition_cmd.extend(['-p', disk])
# Run the partitioning command (all physical partitions in one shot)
retval = self.cmd.run(partition_cmd)
if retval != 0:
raise Exception("Failed partition disk, command: {0}".format(partition_cmd))
# For RPi image we used 'parted' instead of 'sgdisk':
# parted -s $IMAGE_NAME mklabel msdos mkpart primary fat32 1M 30M mkpart primary ext4 30M 100%
# Try to use 'sgdisk -m' to convert GPT to MBR and see whether it works.
if self.install_config.get('partition_type', 'gpt') == 'msdos':
# m - colon separated partitions list
m = ":".join([str(i) for i in range(1,part_idx)])
retval = self.cmd.run(['sgdisk', '-m', m, disk])
if retval != 0:
raise Exception("Failed to setup efi partition")
# Make loop disk partitions available
if 'loop' in disk:
retval = self.cmd.run(['kpartx', '-avs', disk])
if retval != 0:
raise Exception("Failed to rescan partitions of the disk image {}". format(disk))
# Go through l2 entries again and create logical partitions
for l2 in l2entries:
if 'lvs' not in l2:
continue
lvm_present = True
self._create_logical_volumes(l2['path'], l2['vg_name'], l2['lvs'], l2['extensible'])
if lvm_present:
# add lvm2 package to install list
self._add_packages_to_install('lvm2')
# Create partitions_data (needed for mk-setup-grub.sh)
for partition in partitions:
if "mountpoint" in partition:
if partition['mountpoint'] == '/':
partitions_data['root'] = partition['path']
elif partition['mountpoint'] == '/boot':
partitions_data['boot'] = partition['path']
partitions_data['bootdirectory'] = '/'
# If no separate boot partition, then use /boot folder from root partition
if 'boot' not in partitions_data:
partitions_data['boot'] = partitions_data['root']
partitions_data['bootdirectory'] = '/boot/'
# Sort partitions by mountpoint to be able to mount and
# unmount it in proper sequence
partitions.sort(key=lambda p: self.partition_compare(p))
self.install_config['partitions_data'] = partitions_data
def _format_partitions(self):
partitions = self.install_config['partitions']
self.logger.info(partitions)
# Format the filesystem
for partition in partitions:
ptype = self._get_partition_type(partition)
# Do not format BIOS boot partition
if ptype == PartitionType.BIOS:
continue
if ptype == PartitionType.SWAP:
mkfs_cmd = ['mkswap']
else:
mkfs_cmd = ['mkfs', '-t', partition['filesystem']]
if 'fs_options' in partition:
options = re.sub("[^\S]", " ", partition['fs_options']).split()
mkfs_cmd.extend(options)
mkfs_cmd.extend([partition['path']])
retval = self.cmd.run(mkfs_cmd)
if retval != 0:
raise Exception(
"Failed to format {} partition @ {}".format(partition['filesystem'],
partition['path']))
def getfile(self, filename):
"""
Returns absolute filepath by filename.
"""
for dirname in self.install_config['search_path']:
filepath = os.path.join(dirname, filename)
if os.path.exists(filepath):
return filepath
raise Exception("File {} not found in the following directories {}".format(filename, self.install_config['search_path']))