| Current File : //proc/self/root/usr/lib/python3/dist-packages/cloudinit/config/modules.py |
# Copyright (C) 2008-2022 Canonical Ltd.
# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
#
# Author: Chuck Short <chuck.short@canonical.com>
# Author: Juerg Haefliger <juerg.haefliger@hp.com>
#
# This file is part of cloud-init. See LICENSE file for license information.
import copy
import logging
from inspect import signature
from types import ModuleType
from typing import Dict, List, NamedTuple, Optional
from cloudinit import (
config,
importer,
lifecycle,
performance,
type_utils,
util,
)
from cloudinit.distros import ALL_DISTROS
from cloudinit.helpers import ConfigMerger
from cloudinit.reporting.events import ReportEventStack
from cloudinit.settings import FREQUENCIES
from cloudinit.stages import Init
LOG = logging.getLogger(__name__)
# This prefix is used to make it less
# of a chance that when importing
# we will not find something else with the same
# name in the lookup path...
MOD_PREFIX = "cc_"
# List of modules that have removed upstream. This prevents every downstream
# from having to create upgrade scripts to avoid warnings about missing
# modules.
REMOVED_MODULES = [
"cc_emit_upstart", # Removed in 22.2
"cc_refresh_rmc_and_interface.py", # Removed in 23.2
"cc_migrator", # Removed in 24.1
"cc_rightscale_userdata", # Removed in 24.1
]
RENAMED_MODULES = {
"cc_ubuntu_advantage": "cc_ubuntu_pro", # Renamed 24.1
}
class ModuleDetails(NamedTuple):
module: ModuleType
name: str
frequency: str
run_args: List[str]
def form_module_name(name):
canon_name = name.replace("-", "_")
if canon_name.lower().endswith(".py"):
canon_name = canon_name[0 : (len(canon_name) - 3)]
canon_name = canon_name.strip()
if not canon_name:
return None
if not canon_name.startswith(MOD_PREFIX):
canon_name = "%s%s" % (MOD_PREFIX, canon_name)
return canon_name
def validate_module(mod, name):
if (
not hasattr(mod, "meta")
or "frequency" not in mod.meta
or "distros" not in mod.meta
):
raise ValueError(
f"Module '{mod}' with name '{name}' MUST have a 'meta' attribute "
"of type 'MetaSchema'."
)
if mod.meta["frequency"] not in FREQUENCIES:
raise ValueError(
f"Module '{mod}' with name '{name}' has an invalid frequency "
f"{mod.meta['frequency']}."
)
if hasattr(mod, "schema"):
raise ValueError(
f"Module '{mod}' with name '{name}' has a JSON 'schema' attribute "
"defined. Please define schema in cloud-init-schema,json."
)
def _is_active(module_details: ModuleDetails, cfg: dict) -> bool:
activate_by_schema_keys_keys = frozenset(
module_details.module.meta.get("activate_by_schema_keys", {})
)
if not activate_by_schema_keys_keys:
return True
if not activate_by_schema_keys_keys.intersection(cfg.keys()):
return False
return True
class Modules:
def __init__(self, init: Init, cfg_files=None, reporter=None):
self.init = init
self.cfg_files = cfg_files
# Created on first use
self._cached_cfg: Optional[config.Config] = None
if reporter is None:
reporter = ReportEventStack(
name="module-reporter",
description="module-desc",
reporting_enabled=False,
)
self.reporter = reporter
@property
def cfg(self) -> config.Config:
# None check to avoid empty case causing re-reading
if self._cached_cfg is None:
merger = ConfigMerger(
paths=self.init.paths,
datasource=self.init.datasource,
additional_fns=self.cfg_files,
base_cfg=self.init.cfg,
)
self._cached_cfg = merger.cfg
# Only give out a copy so that others can't modify this...
return copy.deepcopy(self._cached_cfg)
def _read_modules(self, name) -> List[Dict]:
"""Read the modules from the config file given the specified name.
Returns a list of module definitions. E.g.,
[
{
"mod": "bootcmd",
"freq": "always",
"args": "some_arg",
}
]
Note that in the default case, only "mod" will be set.
"""
module_list: List[dict] = []
if name not in self.cfg:
return module_list
cfg_mods = self.cfg.get(name)
if not cfg_mods:
return module_list
for item in cfg_mods:
if not item:
continue
if isinstance(item, str):
module_list.append(
{
"mod": item.strip(),
}
)
elif isinstance(item, (list)):
contents = {}
# Meant to fall through...
if len(item) >= 1:
contents["mod"] = item[0].strip()
if len(item) >= 2:
contents["freq"] = item[1].strip()
if len(item) >= 3:
contents["args"] = item[2:]
if contents:
module_list.append(contents)
elif isinstance(item, (dict)):
contents = {}
valid = False
if "name" in item:
contents["mod"] = item["name"].strip()
valid = True
if "frequency" in item:
contents["freq"] = item["frequency"].strip()
if "args" in item:
contents["args"] = item["args"] or []
if contents and valid:
module_list.append(contents)
else:
raise TypeError(
"Failed to read '%s' item in config, unknown type %s"
% (item, type_utils.obj_name(item))
)
return module_list
def _fixup_modules(self, raw_mods) -> List[ModuleDetails]:
"""Convert list of returned from _read_modules() into new format.
Invalid modules and arguments are ignored.
Also ensures that the module has the required meta fields.
"""
mostly_mods = []
for raw_mod in raw_mods:
raw_name = raw_mod["mod"]
freq = raw_mod.get("freq")
run_args = raw_mod.get("args") or []
mod_name = form_module_name(raw_name)
if not mod_name:
continue
if freq and freq not in FREQUENCIES:
lifecycle.deprecate(
deprecated=(
f"Config specified module {raw_name} has an unknown"
f" frequency {freq}"
),
deprecated_version="22.1",
)
# Misconfigured in /etc/cloud/cloud.cfg. Reset so cc_* module
# default meta attribute "frequency" value is used.
freq = None
if mod_name in RENAMED_MODULES:
lifecycle.deprecate(
deprecated=(
f"Module has been renamed from {mod_name} to "
f"{RENAMED_MODULES[mod_name]}. Update any"
" references in /etc/cloud/cloud.cfg"
),
deprecated_version="24.1",
)
mod_name = RENAMED_MODULES[mod_name]
mod_locs, looked_locs = importer.find_module(
mod_name, ["", type_utils.obj_name(config)], ["handle"]
)
if not mod_locs:
if mod_name in REMOVED_MODULES:
LOG.info(
"Module `%s` has been removed from cloud-init. "
"It may be removed from `/etc/cloud/cloud.cfg`.",
mod_name[3:], # [3:] to remove 'cc_'
)
else:
LOG.warning(
"Could not find module named %s (searched %s)",
mod_name,
looked_locs,
)
continue
mod = importer.import_module(mod_locs[0])
validate_module(mod, raw_name)
if freq is None:
# Use cc_* module default setting since no cloud.cfg overrides
freq = mod.meta["frequency"]
mostly_mods.append(
ModuleDetails(
module=mod,
name=raw_name,
frequency=freq,
run_args=run_args,
)
)
return mostly_mods
def _run_modules(self, mostly_mods: List[ModuleDetails]):
cc = self.init.cloudify()
# Return which ones ran
# and which ones failed + the exception of why it failed
failures = []
which_ran = []
for mod, name, freq, args in mostly_mods:
try:
LOG.debug(
"Running module %s (%s) with frequency %s", name, mod, freq
)
# Mark it as having started running
which_ran.append(name)
# This name will affect the semaphore name created
run_name = f"config-{name}"
desc = "running %s with frequency %s" % (run_name, freq)
myrep = ReportEventStack(
name=run_name, description=desc, parent=self.reporter
)
func_args = {
"name": name,
"cfg": self.cfg,
"cloud": cc,
"args": args,
}
with myrep:
func_signature = signature(mod.handle)
func_params = func_signature.parameters
if len(func_params) == 5:
lifecycle.deprecate(
deprecated="Config modules with a `log` parameter",
deprecated_version="23.2",
)
func_args.update({"log": LOG})
with performance.Timed("", log_mode="skip") as timer:
ran, _r = cc.run(
run_name, mod.handle, func_args, freq=freq
)
if ran:
myrep.message = (
f"{run_name} ran successfully and "
f"took {timer.delta:.3f} seconds"
)
else:
myrep.message = "%s previously ran" % run_name
except Exception as e:
util.logexc(LOG, "Running module %s (%s) failed", name, mod)
failures.append((name, e))
return (which_ran, failures)
def run_single(self, mod_name, args=None, freq=None):
# Form the users module 'specs'
mod_to_be = {
"mod": mod_name,
"args": args,
"freq": freq,
}
# Now resume doing the normal fixups and running
raw_mods = [mod_to_be]
mostly_mods = self._fixup_modules(raw_mods)
return self._run_modules(mostly_mods)
def run_section(self, section_name):
"""Runs all modules in the given section.
section_name - One of the modules lists as defined in
/etc/cloud/cloud.cfg. One of:
- cloud_init_modules
- cloud_config_modules
- cloud_final_modules
"""
raw_mods = self._read_modules(section_name)
mostly_mods = self._fixup_modules(raw_mods)
distro_name = self.init.distro.name
skipped = []
forced = []
overridden = self.cfg.get("unverified_modules", [])
inapplicable_mods = []
active_mods = []
for module_details in mostly_mods:
(mod, name, _freq, _args) = module_details
if mod is None:
continue
worked_distros = mod.meta["distros"]
if not _is_active(module_details, self.cfg):
inapplicable_mods.append(name)
continue
# Skip only when the following conditions are all met:
# - distros are defined in the module != ALL_DISTROS
# - the current d_name isn't in distros
# - and the module is unverified and not in the unverified_modules
# override list
if worked_distros and worked_distros != [ALL_DISTROS]:
if distro_name not in worked_distros:
if name not in overridden:
skipped.append(name)
continue
forced.append(name)
active_mods.append([mod, name, _freq, _args])
if inapplicable_mods:
LOG.info(
"Skipping modules '%s' because no applicable config "
"is provided.",
",".join(inapplicable_mods),
)
if skipped:
LOG.info(
"Skipping modules '%s' because they are not verified "
"on distro '%s'. To run anyway, add them to "
"'unverified_modules' in config.",
",".join(skipped),
distro_name,
)
if forced:
LOG.info("running unverified_modules: '%s'", ", ".join(forced))
return self._run_modules(active_mods)